agentic-scorecard 0.1.0 → 0.2.0

package/benchmark/v0.2/controls/context.yaml

@@ -0,0 +1,51 @@
+dimension: context
+controls:
+  - id: ADRB-CTX-001
+    level: 1
+    title: Agent guidance is discoverable
+    outcome: A contributor or agent can locate the repository's authoritative operating guidance.
+    risk: Agents infer repository rules from generic practice or stale neighboring files.
+    evidence:
+      - type: path_any
+        scope: repository
+        patterns: [AGENTS.md, CLAUDE.md, .github/copilot-instructions.md, .agents/**, .claude/**]
+    remediation: Add a tracked, non-empty agent entry point that names the authoritative guidance.
+  - id: ADRB-CTX-002
+    level: 2
+    title: Guidance scope and precedence are explicit
+    outcome: Instructions state their scope, precedence, important constraints, and architecture boundaries.
+    risk: Conflicting or nested guidance produces plausible changes that violate local invariants.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files:
+          [AGENTS.md, CLAUDE.md, .github/copilot-instructions.md, .agents/**/*.md, .claude/**/*.md]
+        terms: [scope, precedence, architecture, must, constraint]
+        min_terms: 2
+    remediation: Document instruction scope, precedence, non-negotiable constraints, and where deeper guidance lives.
+  - id: ADRB-CTX-003
+    level: 3
+    title: Agent guidance integrity is mechanically checked
+    outcome: Automated validation detects missing, broken, stale, or structurally invalid agent guidance.
+    risk: Critical instructions silently drift while agents continue treating them as authoritative.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files: [.github/workflows/**, scripts/**, package.json, Makefile]
+        terms: [validate, lint, stale, broken link, integrity, check]
+        min_terms: 1
+        required_any_terms:
+          [agent docs, agent instructions, AGENTS.md, CLAUDE.md, copilot instructions]
+    remediation: Add a CI check for required agent files, links, ownership, and structural invariants.
+  - id: ADRB-CTX-004
+    level: 4
+    title: Guidance effectiveness drives improvement
+    outcome: Retrieval failures, repeated instruction errors, and correction latency are trended and improve guidance.
+    risk: Documentation volume grows without evidence that agents find and apply the right context.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide time-series guidance-quality outcomes and a recent instruction change driven by them.
+    remediation: Measure context retrieval and recurring instruction failures, then update guidance from the results.

package/benchmark/v0.2/controls/environment.yaml

@@ -0,0 +1,70 @@
+dimension: environment
+controls:
+  - id: ADRB-ENV-001
+    level: 1
+    title: Development setup is documented
+    outcome: Contributors can identify supported runtime prerequisites and the canonical setup path.
+    risk: Agents improvise dependencies or run commands in an incompatible environment.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files: [README.md, CONTRIBUTING.md, docs/setup/**, docs/development/**]
+        terms: [install, setup, runtime, prerequisites, dependencies, development]
+        min_terms: 2
+    remediation: Document supported runtimes, prerequisites, installation, and the first verification command.
+  - id: ADRB-ENV-002
+    level: 2
+    title: Dependencies and toolchain are reproducibly pinned
+    outcome: The repository records deterministic dependency resolution and a supported runtime or container definition.
+    risk: Local and CI behavior diverge as dependency or runtime versions float.
+    evidence:
+      - type: path_any
+        scope: repository
+        patterns:
+          [
+            package-lock.json,
+            pnpm-lock.yaml,
+            yarn.lock,
+            uv.lock,
+            poetry.lock,
+            Pipfile.lock,
+            Cargo.lock,
+            go.sum,
+          ]
+      - type: path_any
+        scope: repository
+        patterns:
+          [
+            .node-version,
+            .nvmrc,
+            .python-version,
+            mise.toml,
+            .tool-versions,
+            Dockerfile,
+            .devcontainer/**,
+          ]
+    remediation: Commit a lockfile and a supported runtime, toolchain, or development-container declaration.
+  - id: ADRB-ENV-003
+    level: 3
+    title: CI recreates the canonical environment
+    outcome: Continuous integration installs from locked inputs and runs canonical verification in a clean environment.
+    risk: A local-only setup passes while clean automation fails or uses materially different dependencies.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files: [.github/workflows/**, .gitlab-ci.yml, Jenkinsfile, azure-pipelines.yml]
+        terms: [npm ci, frozen-lockfile, uv sync, lockfile, checkout, test, typecheck]
+        min_terms: 2
+    remediation: Make CI install locked dependencies from a clean checkout and run the documented verification path.
+  - id: ADRB-ENV-004
+    level: 4
+    title: Environment reliability drives improvement
+    outcome: Setup failures, cache misses, drift, and reproducibility incidents are trended and reduced.
+    risk: Environment friction and nondeterminism remain anecdotal and repeatedly consume agent time.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide environment reliability trends and a recent setup or CI improvement driven by them.
+    remediation: Track clean-setup and CI-environment outcomes, then prioritize the largest sources of drift.

package/benchmark/v0.2/controls/governance.yaml

@@ -0,0 +1,60 @@
+dimension: governance
+controls:
+  - id: ADRB-GOV-001
+    level: 1
+    title: Contribution and review expectations are documented
+    outcome: Contributors can find non-empty guidance describing how changes are proposed and reviewed.
+    risk: Agents submit changes without a shared review or accountability process.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files:
+          [
+            CONTRIBUTING.md,
+            .github/pull_request_template.md,
+            .github/PULL_REQUEST_TEMPLATE/**,
+            docs/contributing/**,
+          ]
+        terms: [pull request, review, approval, testing, contribution]
+        min_terms: 2
+    remediation: Publish contribution guidance covering change proposals, verification, reviewers, and approvals.
+  - id: ADRB-GOV-002
+    level: 2
+    title: Ownership and merge authority are declared
+    outcome: The tracked repository assigns review ownership and states who may approve or merge changes.
+    risk: Sensitive or cross-cutting changes reach reviewers without accountable ownership.
+    evidence:
+      - type: path_any
+        scope: repository
+        patterns: [.github/CODEOWNERS, CODEOWNERS]
+      - type: content_terms
+        scope: repository
+        files:
+          [CONTRIBUTING.md, GOVERNANCE.md, .github/pull_request_template.md, docs/governance/**]
+        terms: [owner, reviewer, approval, merge authority, required review]
+        min_terms: 2
+    remediation: Commit ownership rules and document reviewer, approval, merge, and exception authority.
+  - id: ADRB-GOV-003
+    level: 3
+    title: Protected merge governance is enforced
+    outcome: Platform rules enforce reviews, required checks, protected branches, and controlled exceptions.
+    risk: Repository-side declarations can be bypassed because platform enforcement is absent or too broad.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: platform
+        prompt: Show protected branch or ruleset coverage, required reviews and checks, merge authority, bypass actors, and exception records.
+    remediation: Enforce branch or merge rules for relevant branches and restrict, record, and review bypasses.
+  - id: ADRB-GOV-004
+    level: 4
+    title: Governance outcomes drive improvement
+    outcome: Review escapes, bypasses, rollback causes, and approval latency are trended and improve governance.
+    risk: Governance becomes ceremony without evidence that it catches risk efficiently.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide governance outcome trends and a recent review or merge-policy improvement driven by them.
+    remediation: Trend governance outcomes by risk class and tune ownership, review depth, and exceptions.

package/benchmark/v0.2/controls/learning.yaml

@@ -0,0 +1,48 @@
+dimension: learning
+controls:
+  - id: ADRB-LRN-001
+    level: 1
+    title: Agent corrections have a durable home
+    outcome: Review corrections and post-merge lessons are recorded in a tracked repository artifact.
+    risk: Every agent session repeats the same repository-specific mistakes.
+    evidence:
+      - type: path_any
+        scope: repository
+        patterns: [.ai/lessons.md, docs/lessons/**, docs/decisions/**, ADR/**, docs/adr/**]
+    remediation: Create a concise tracked correction log with context, rule, owner, and date.
+  - id: ADRB-LRN-002
+    level: 2
+    title: Knowledge promotion and ownership are documented
+    outcome: Guidance defines when corrections become instructions, skills, decisions, or maintained references.
+    risk: The lesson store becomes a stale archive rather than usable agent knowledge.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files: [AGENTS.md, .ai/lessons.md, docs/knowledge/**, .agents/skills/**, .claude/skills/**]
+        terms: [curation, promote, lesson, correction, stale, owner, deduplication]
+        min_terms: 3
+    remediation: Define curation cadence, promotion rules, ownership, deduplication, and stale-content handling.
+  - id: ADRB-LRN-003
+    level: 3
+    title: Knowledge curation is mechanically enforced
+    outcome: Automation detects stale, duplicate, contradictory, or unpromoted agent knowledge.
+    risk: Knowledge quality relies on memory and quietly deteriorates as the repository evolves.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files: [.github/workflows/**, scripts/**, package.json, Makefile]
+        terms: [curator, curation, knowledge lint, stale, contradiction, duplicate]
+        min_terms: 2
+    remediation: Automate mechanical checks and require periodic judgement over knowledge quality.
+  - id: ADRB-LRN-004
+    level: 4
+    title: Knowledge effectiveness drives improvement
+    outcome: Recurring mistakes, stale guidance, retrieval success, and correction latency are trended and improve curation.
+    risk: Knowledge grows without evidence that it changes agent behavior.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide recurring-error and knowledge-quality trends plus a recent curation improvement driven by them.
+    remediation: Measure whether promoted knowledge prevents repeated mistakes and prioritize weak guidance.

package/benchmark/v0.2/controls/observability.yaml

@@ -0,0 +1,76 @@
+dimension: observability
+controls:
+  - id: ADRB-OBS-001
+    level: 1
+    title: Agent run records are defined
+    outcome: A tracked schema or guide defines how material agent activity can be reconstructed.
+    risk: Teams cannot explain what happened after a poor change or security concern.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files:
+          [
+            docs/observability/**,
+            docs/agentic/**,
+            evals/**,
+            evaluations/**,
+            observability/**,
+            .agentic/run-envelope.*,
+          ]
+        terms: [task, actor, harness, result, timestamp, evidence]
+        min_terms: 3
+    remediation: Define a privacy-safe run record containing task, harness, result, timing, and evidence references.
+  - id: ADRB-OBS-002
+    level: 2
+    title: Run envelope and outcome taxonomy are documented
+    outcome: Run guidance covers authority, tools, budget, stop reason, verification, and outcome classification.
+    risk: Comparisons confound model, tooling, permissions, task risk, and checkout state.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files:
+          [
+            docs/observability/**,
+            docs/agentic/**,
+            evals/**,
+            evaluations/**,
+            observability/**,
+            .agentic/run-envelope.*,
+          ]
+        terms:
+          [
+            task class,
+            tool,
+            permission,
+            budget,
+            stop reason,
+            outcome,
+            harness version,
+            verification,
+          ]
+        min_terms: 4
+    remediation: Adopt a common run envelope and outcome taxonomy with privacy and retention rules.
+  - id: ADRB-OBS-003
+    level: 3
+    title: Outcome evidence is captured consistently
+    outcome: Material agent runs automatically emit retained metadata and verification outcomes.
+    risk: Only memorable successes and failures shape policy, creating selection bias.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: platform
+        prompt: Show automated run-envelope capture, completeness monitoring, retention controls, and access controls.
+    remediation: Emit structured run and outcome records automatically while minimizing sensitive content.
+  - id: ADRB-OBS-004
+    level: 4
+    title: Agent outcomes drive autonomy decisions
+    outcome: Quality, escape rate, cycle time, cost, intervention, and rollback trends change autonomy or harness policy.
+    risk: Token use or completion rate becomes a misleading proxy for engineering value and safety.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide outcome trends segmented by task and risk class and an autonomy or harness decision they changed.
+    remediation: Evaluate quality and total effort by task class and use results to narrow or expand autonomy.

package/benchmark/v0.2/controls/resilience.yaml

@@ -0,0 +1,60 @@
+dimension: resilience
+controls:
+  - id: ADRB-RES-001
+    level: 1
+    title: Changes have a documented recovery path
+    outcome: Contributors can locate actionable rollback, revert, recovery, or incident guidance.
+    risk: A harmful agent change persists because recovery is improvised under pressure.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files:
+          [
+            README.md,
+            CONTRIBUTING.md,
+            docs/recovery/**,
+            docs/operations/**,
+            runbooks/**,
+            .github/**,
+          ]
+        terms: [rollback, revert, recovery, incident, restore]
+        min_terms: 2
+    remediation: Document how to stop work, revert code, recover affected state, and escalate an incident.
+  - id: ADRB-RES-002
+    level: 2
+    title: Failure containment and stop conditions are documented
+    outcome: Agent guidance defines mutation limits, budgets, stop triggers, escalation, and recovery ownership.
+    risk: A looping or confused agent continues spending or mutating beyond the intended scope.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files:
+          [AGENTS.md, SECURITY.md, docs/agentic/**, docs/recovery/**, runbooks/**, templates/**]
+        terms:
+          [stop condition, rollback, isolation, budget, kill switch, escalation, recovery owner]
+        min_terms: 3
+    remediation: Add bounded scope, budget, retry, stop, escalation, and recovery requirements to task preflight.
+  - id: ADRB-RES-003
+    level: 3
+    title: Containment and recovery controls are exercised
+    outcome: Cancellation, credential revocation, rollback, timeout, and partial-failure recovery are tested.
+    risk: Recovery mechanisms exist on paper but fail when the harness behaves unexpectedly.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: organization
+        prompt: Show recent automated tests or exercises for stop, timeout, revocation, rollback, and partial failure.
+    remediation: Add failure-injection tests and periodic exercises for containment and reversible recovery.
+  - id: ADRB-RES-004
+    level: 4
+    title: Recovery outcomes drive improvement
+    outcome: Time-to-detect, time-to-stop, rollback success, impact, and recurrence trends improve containment.
+    risk: The same failure patterns repeat because recovery performance and residual impact are unknown.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide recovery outcome trends and a recent containment or rollback improvement driven by them.
+    remediation: Trend containment and recovery outcomes and feed them into autonomy and tool design.

package/benchmark/v0.2/controls/security.yaml

@@ -0,0 +1,71 @@
+dimension: security
+controls:
+  - id: ADRB-SEC-001
+    level: 1
+    title: Security reporting and secret handling are documented
+    outcome: Contributors can find security contacts plus rules for credentials and sensitive data.
+    risk: Agents expose secrets or sensitive data and responders lack a defined reporting path.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files: [SECURITY.md, .github/SECURITY.md, docs/security/**]
+        terms: [report, vulnerability, secret, credential, sensitive]
+        min_terms: 2
+    remediation: Publish a security policy covering reporting, secrets, sensitive data, and agent-specific risks.
+  - id: ADRB-SEC-002
+    level: 2
+    title: Agent data and autonomy boundaries are explicit
+    outcome: Agent preflight guidance binds permitted data, tools, mutations, approvals, and stop conditions.
+    risk: Operators grant more data or authority than the task requires.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files: [SECURITY.md, AGENTS.md, docs/security/**, docs/agentic/**, templates/**]
+        terms:
+          [
+            data classification,
+            allowed tools,
+            mutation,
+            budget,
+            human approval,
+            autonomy,
+            stop condition,
+          ]
+        min_terms: 3
+    remediation: Add a task preflight that binds data, tools, write scope, budget, approvals, and stop conditions.
+  - id: ADRB-SEC-003
+    level: 3
+    title: Automated agent security safeguards are configured
+    outcome: Repository automation checks secrets and defines handling for untrusted agent input.
+    risk: Credential exposure or prompt-borne instructions can pass through an otherwise healthy pipeline.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files: [.github/workflows/**, .gitlab-ci.yml, scripts/**, SECURITY.md, docs/security/**]
+        terms: [secret scan, gitleaks, trufflehog, prompt injection, untrusted content, sandbox]
+        min_terms: 2
+    remediation: Configure secret scanning and explicit untrusted-content or prompt-injection safeguards with tests.
+  - id: ADRB-SEC-005
+    level: 3
+    title: High-impact agent actions are technically gated
+    outcome: External authority controls prevent unapproved high-impact actions and record controlled exceptions.
+    risk: Repository instructions alone cannot prevent credential misuse or irreversible external actions.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: platform
+        prompt: Show authorization gates, scoped credentials, approval enforcement, audit records, and controlled bypasses for high-impact agent actions.
+    remediation: Enforce least-privileged credentials, approval gates, auditability, and restricted exception handling.
+  - id: ADRB-SEC-004
+    level: 4
+    title: Agent security outcomes drive improvement
+    outcome: Denials, near misses, incidents, exceptions, and adversarial exercises are trended and improve controls.
+    risk: Controls appear complete while effectiveness against real failure modes remains unknown.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide agent-security outcome trends or exercise results and a recent control change driven by them.
+    remediation: Run adversarial exercises, trend security outcomes, and update boundaries from the results.

package/benchmark/v0.2/controls/specification.yaml

@@ -0,0 +1,62 @@
+dimension: specification
+controls:
+  - id: ADRB-SPC-001
+    level: 1
+    title: Work has a durable objective
+    outcome: Engineering work starts from a tracked issue, request, or specification artifact.
+    risk: Agents optimize an inferred problem and completion cannot be judged consistently.
+    evidence:
+      - type: path_any
+        scope: repository
+        patterns:
+          [.github/ISSUE_TEMPLATE/**, docs/requirements/**, specs/**, .ai/specs/**, .kiro/specs/**]
+    remediation: Adopt a tracked work-request or specification template with objective, owner, and outcome.
+  - id: ADRB-SPC-002
+    level: 2
+    title: Acceptance criteria and constraints are documented
+    outcome: Work artifacts state testable acceptance criteria plus material constraints or exclusions.
+    risk: An implementation can appear plausible without satisfying a shared definition of done.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files:
+          [.github/ISSUE_TEMPLATE/**, specs/**, .ai/specs/**, .kiro/specs/**, docs/requirements/**]
+        terms: [acceptance criteria, requirements, constraints, non-goals, verification]
+        min_terms: 2
+    remediation: Add testable outcomes, constraints, exclusions, and verification expectations to work artifacts.
+  - id: ADRB-SPC-003
+    level: 3
+    title: Specification traceability is mechanically checked
+    outcome: Automation checks that applicable changes and verification trace to an approved work artifact.
+    risk: Specifications become optional prose disconnected from implementation and tests.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files: [.github/workflows/**, scripts/**, package.json, Makefile]
+        terms: [acceptance criteria, traceability, requirements coverage, spec check, task id]
+        min_terms: 2
+    remediation: Add a deterministic traceability check from change to work artifact and verification record.
+  - id: ADRB-SPC-005
+    level: 3
+    title: Specification traceability is required before merge
+    outcome: Repository merge policy requires the applicable traceability check or equivalent review evidence.
+    risk: A configured check can be skipped or bypassed when delivery pressure rises.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: platform
+        prompt: Show that the traceability check or equivalent review requirement is an enforced merge gate with controlled exceptions.
+    remediation: Require the traceability check in merge policy and restrict, record, and review bypasses.
+  - id: ADRB-SPC-004
+    level: 4
+    title: Specification effectiveness drives improvement
+    outcome: Rework and escaped ambiguity attributable to specifications are trended and improve the process.
+    risk: Specification ceremony grows without reducing misunderstanding or rework.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide specification-related rework or ambiguity trends and a recent process change driven by them.
+    remediation: Classify rework causes and evolve specification practices using outcome evidence.

package/benchmark/v0.2/controls/testing.yaml

@@ -0,0 +1,60 @@
+dimension: testing
+controls:
+  - id: ADRB-TST-001
+    level: 1
+    title: Automated tests exist
+    outcome: The tracked repository contains executable tests for at least one important behavior.
+    risk: Agent changes rely on plausibility or manual inspection alone.
+    evidence:
+      - type: path_any
+        scope: repository
+        patterns: ['**/*.test.*', '**/*.spec.*', tests/**, test/**]
+    remediation: Add a fast automated test for a critical behavior and document how to run it.
+  - id: ADRB-TST-002
+    level: 2
+    title: Verification layers and canonical commands are documented
+    outcome: Contributors can identify applicable test layers and the canonical verification path.
+    risk: Agents run a convenient subset and overstate confidence in the result.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files: [README.md, CONTRIBUTING.md, AGENTS.md, docs/testing/**, package.json, Makefile]
+        terms: [unit test, integration test, end-to-end, e2e, typecheck, lint, test]
+        min_terms: 2
+    remediation: Define the test taxonomy and one canonical command that matches CI.
+  - id: ADRB-TST-003
+    level: 3
+    title: CI executes the required verification layers
+    outcome: Repository automation runs appropriate tests and static checks from a clean state.
+    risk: Documented verification can be omitted from automation or run in an unrepresentative environment.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files: [.github/workflows/**, .gitlab-ci.yml, Jenkinsfile, azure-pipelines.yml]
+        terms: [test, typecheck, lint, coverage, e2e, integration]
+        min_terms: 2
+    remediation: Configure CI to run the applicable verification layers from a clean environment.
+  - id: ADRB-TST-005
+    level: 3
+    title: Verification is enforced before merge
+    outcome: Merge policy requires the applicable CI checks and restricts bypasses.
+    risk: Healthy CI configuration remains optional under delivery pressure.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: platform
+        prompt: Show required status checks or equivalent merge gates, covered branches, bypass actors, and exception controls.
+    remediation: Make appropriate CI checks required merge gates and restrict, record, and review bypasses.
+  - id: ADRB-TST-004
+    level: 4
+    title: Verification effectiveness drives improvement
+    outcome: Escaped defects, flakiness, and other signal-quality outcomes are trended and improve testing.
+    risk: Test count and coverage rise while meaningful defects still escape.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide verification outcome trends by task class and a recent test-strategy improvement driven by them.
+    remediation: Trend escaped defects and signal quality, then invest in the weakest verification layers.

package/benchmark/v0.2/controls/tooling.yaml

@@ -0,0 +1,58 @@
+dimension: tooling
+controls:
+  - id: ADRB-TOL-001
+    level: 1
+    title: Supported agent tools are declared
+    outcome: The tracked repository identifies supported agent interfaces or tool configuration.
+    risk: Agents discover tools opportunistically and use unsupported or unsafe paths.
+    evidence:
+      - type: path_any
+        scope: repository
+        patterns:
+          [
+            .codex/**,
+            .claude/**,
+            .agents/**,
+            .cursor/**,
+            .github/copilot-instructions.md,
+            mcp.json,
+            .mcp.json,
+          ]
+    remediation: Commit a non-empty declaration of supported agent surfaces and their intended use.
+  - id: ADRB-TOL-002
+    level: 2
+    title: Tool scope and failure behavior are documented
+    outcome: Tool guidance defines authority, validation, success signals, failure handling, and approved fallback.
+    risk: Agents bypass failures, widen scope, or mistake partial results for success.
+    evidence:
+      - type: content_terms
+        scope: repository
+        files:
+          [AGENTS.md, CLAUDE.md, .agents/**/*.md, .claude/**/*.md, docs/agentic/**, docs/tools/**]
+        terms: [authority, permission, read-only, validation, failure, fallback, approval]
+        min_terms: 3
+    remediation: Document each high-impact tool's authority, inputs, validation, failure mode, and fallback.
+  - id: ADRB-TOL-003
+    level: 3
+    title: High-impact tool authority is technically bounded and audited
+    outcome: High-impact tools enforce least privilege, authorization, structured validation, and audit records.
+    risk: Prompt instructions become the only barrier between an agent and an irreversible action.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: platform
+        prompt: Show technical least-privilege enforcement, authorization tests, validation, audit records, and fail-closed behavior for high-impact tools.
+    remediation: Enforce scoped credentials, authorization, schema validation, audit logging, and deny-by-default behavior.
+  - id: ADRB-TOL-004
+    level: 4
+    title: Tool reliability drives improvement
+    outcome: Tool success, partial failure, denial, latency, and unsafe-fallback attempts are trended and improved.
+    risk: Tooling quality and risky workarounds remain anecdotal until an incident occurs.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide tool reliability trends and a recent tool or fallback improvement driven by them.
+    remediation: Trend tool outcomes by task and risk class and improve weak or bypassed interfaces.

package/benchmark/v0.2/report-schema.json

@@ -0,0 +1,47 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agentic-readiness.dev/schema/v0.2/report.json",
+  "title": "ADRB v0.2 assessment report",
+  "type": "object",
+  "required": [
+    "schema_version",
+    "benchmark",
+    "target",
+    "assessed_at",
+    "score",
+    "evidence_summary",
+    "dimensions",
+    "controls",
+    "profiles",
+    "readiness",
+    "limitations"
+  ],
+  "properties": {
+    "schema_version": { "const": "0.2.0" },
+    "benchmark": {
+      "type": "object",
+      "required": ["id", "version"],
+      "properties": { "version": { "const": "0.2.0" } }
+    },
+    "target": {
+      "type": "object",
+      "required": [
+        "repository",
+        "profile",
+        "scope",
+        "git_head",
+        "git_remote",
+        "working_tree_dirty"
+      ],
+      "properties": { "scope": { "enum": ["tracked", "workspace"] } }
+    },
+    "assessed_at": { "type": "string", "format": "date-time" },
+    "score": { "type": "object" },
+    "evidence_summary": { "type": "object" },
+    "dimensions": { "type": "array", "minItems": 10, "maxItems": 10 },
+    "controls": { "type": "array" },
+    "profiles": { "type": "array" },
+    "readiness": { "type": "object" },
+    "limitations": { "type": "array", "items": { "type": "string" } }
+  }
+}