agentic-scorecard 0.3.1 → 0.4.0

package/benchmark/v0.4/controls/environment.yaml

@@ -0,0 +1,93 @@
+dimension: environment
+controls:
+  - id: ADRB-ENV-001
+    level: 1
+    title: Development setup is documented
+    outcome: Contributors can identify supported runtime prerequisites and the canonical setup path.
+    risk: Agents improvise dependencies or run commands in an incompatible environment.
+    allow_agent_evidence: true
+    agent_evidence_scopes: [repository]
+    evidence:
+      - type: content_terms
+        scope: repository
+        files:
+          [
+            README.md,
+            CONTRIBUTING.md,
+            AGENTS.md,
+            Agents.md,
+            agents.md,
+            .ai/AGENTS.md,
+            .ai/Agents.md,
+            .ai/constitution.md,
+            .ai/instructions/**/*.md,
+            .ai/rules/**/*.md,
+            .ai/skills/**/*.md,
+            .agents/**/*.md,
+            docs/setup/**,
+            docs/development/**,
+          ]
+        terms: [install, setup, runtime, prerequisites, dependencies, development]
+        min_terms: 2
+        max_span_lines: 120
+        max_files_per_pattern: 50
+    remediation: Document supported runtimes, prerequisites, installation, and the first verification command.
+  - id: ADRB-ENV-002
+    level: 2
+    title: Dependencies and toolchain are reproducibly pinned
+    outcome: The repository records deterministic dependency resolution and a supported runtime or container definition.
+    risk: Local and CI behavior diverge as dependency or runtime versions float.
+    evidence:
+      - type: path_any
+        scope: repository
+        patterns:
+          [
+            package-lock.json,
+            pnpm-lock.yaml,
+            yarn.lock,
+            uv.lock,
+            poetry.lock,
+            Pipfile.lock,
+            Cargo.lock,
+            go.sum,
+          ]
+      - type: path_any
+        scope: repository
+        patterns:
+          [
+            .node-version,
+            .nvmrc,
+            .python-version,
+            mise.toml,
+            .tool-versions,
+            Dockerfile,
+            .devcontainer/**,
+          ]
+    remediation: Commit a lockfile and a supported runtime, toolchain, or development-container declaration.
+  - id: ADRB-ENV-003
+    level: 3
+    title: CI recreates the canonical environment
+    outcome: Continuous integration installs from locked inputs and runs canonical verification in a clean environment.
+    risk: A local-only setup passes while clean automation fails or uses materially different dependencies.
+    allow_agent_evidence: true
+    agent_evidence_scopes: [repository]
+    evidence:
+      - type: ci_command
+        scope: repository
+        providers: []
+        tools: []
+        min_tools: 3
+        tool_match_mode: same-execution
+    remediation: Make CI install locked dependencies from a clean checkout and run the documented verification path.
+  - id: ADRB-ENV-004
+    level: 4
+    title: Environment reliability drives improvement
+    outcome: Setup failures, cache misses, drift, and reproducibility incidents are trended and reduced.
+    risk: Environment friction and nondeterminism remain anecdotal and repeatedly consume agent time.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide environment reliability trends and a recent setup or CI improvement driven by them.
+    remediation: Track clean-setup and CI-environment outcomes, then prioritize the largest sources of drift.

package/benchmark/v0.4/controls/governance.yaml

@@ -0,0 +1,132 @@
+dimension: governance
+controls:
+  - id: ADRB-GOV-001
+    level: 1
+    title: Contribution and review expectations are documented
+    outcome: Contributors can find non-empty guidance describing how changes are proposed and reviewed.
+    risk: Agents submit changes without a shared review or accountability process.
+    allow_agent_evidence: true
+    agent_evidence_scopes: [repository]
+    evidence:
+      - type: content_terms
+        scope: repository
+        files:
+          [
+            CONTRIBUTING.md,
+            AGENTS.md,
+            Agents.md,
+            agents.md,
+            .ai/AGENTS.md,
+            .ai/Agents.md,
+            .ai/constitution.md,
+            .ai/instructions/**/*.md,
+            .ai/rules/**/*.md,
+            .ai/skills/**/*.md,
+            .agents/**/*.md,
+            docs/contributing/**,
+            docs/agentic/**,
+          ]
+        terms: [pull request, review, approval, testing, contribution, before merge]
+        min_terms: 2
+        required_any_terms: [pull request, review, approval, before merge]
+        max_span_lines: 100
+        max_files_per_pattern: 50
+    remediation: Publish contribution guidance covering change proposals, verification, reviewers, and approvals.
+  - id: ADRB-GOV-002
+    level: 2
+    title: Ownership, approval, and merge authority are declared
+    outcome: The tracked repository assigns review ownership and states which accountable human roles may approve and merge changes.
+    risk: Sensitive or cross-cutting changes reach reviewers without accountable ownership.
+    allow_agent_evidence: true
+    agent_evidence_scopes: [repository]
+    evidence:
+      - type: ownership_map
+        scope: repository
+        patterns:
+          [
+            CODEOWNERS,
+            OWNERS,
+            OWNERS.md,
+            MAINTAINERS,
+            MAINTAINERS.md,
+            GOVERNANCE.md,
+            docs/governance/**,
+            docs/ownership/**,
+            .ai/ownership/**,
+          ]
+      - type: content_groups
+        scope: repository
+        files:
+          [
+            CONTRIBUTING.md,
+            GOVERNANCE.md,
+            AGENTS.md,
+            Agents.md,
+            agents.md,
+            .ai/AGENTS.md,
+            .ai/Agents.md,
+            .ai/constitution.md,
+            .ai/instructions/**/*.md,
+            .ai/rules/**/*.md,
+            .ai/skills/**/*.md,
+            .agents/**/*.md,
+            docs/governance/**,
+            docs/agentic/**,
+          ]
+        groups:
+          - id: approval-authority
+            terms:
+              [
+                owner may approve,
+                owners may approve,
+                reviewer may approve,
+                reviewers may approve,
+                maintainer may approve,
+                maintainers may approve,
+                reviewer provides approval,
+                reviewer provides human approval,
+                reviewer approval,
+                maintainer approval is required,
+                maintainer approval required,
+              ]
+          - id: merge-authority
+            terms:
+              [
+                owner may merge,
+                owners may merge,
+                reviewer may merge,
+                reviewers may merge,
+                maintainer may merge,
+                maintainers may merge,
+                maintainer has merge authority,
+                maintainers have merge authority,
+                maintainers retain merge authority,
+              ]
+        min_groups: 2
+        max_span_lines: 100
+        max_files_per_pattern: 50
+    remediation: Commit ownership rules and document who reviews, who approves, who merges, and who controls exceptions.
+  - id: ADRB-GOV-003
+    level: 3
+    title: Protected merge governance is enforced
+    outcome: Platform rules enforce reviews, required checks, protected branches, and controlled exceptions.
+    risk: Repository-side declarations can be bypassed because platform enforcement is absent or too broad.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: platform
+        prompt: Show protected branch or ruleset coverage, required reviews and checks, merge authority, bypass actors, and exception records.
+    remediation: Enforce branch or merge rules for relevant branches and restrict, record, and review bypasses.
+  - id: ADRB-GOV-004
+    level: 4
+    title: Governance outcomes drive improvement
+    outcome: Review escapes, bypasses, rollback causes, and approval latency are trended and improve governance.
+    risk: Governance becomes ceremony without evidence that it catches risk efficiently.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide governance outcome trends and a recent review or merge-policy improvement driven by them.
+    remediation: Trend governance outcomes by risk class and tune ownership, review depth, and exceptions.

package/benchmark/v0.4/controls/learning.yaml

@@ -0,0 +1,78 @@
+dimension: learning
+controls:
+  - id: ADRB-LRN-001
+    level: 1
+    title: Agent corrections have a durable home
+    outcome: Review corrections and post-merge lessons are recorded in a tracked repository artifact.
+    risk: Every agent session repeats the same repository-specific mistakes.
+    allow_agent_evidence: true
+    agent_evidence_scopes: [repository]
+    evidence:
+      - type: path_any
+        scope: repository
+        patterns:
+          [
+            .ai/lessons.md,
+            .ai/memory/**,
+            .agents/memory/**,
+            docs/lessons/**,
+            docs/decisions/**,
+            ADR/**,
+            docs/adr/**,
+          ]
+    remediation: Create a concise tracked correction log with context, rule, owner, and date.
+  - id: ADRB-LRN-002
+    level: 2
+    title: Knowledge promotion and ownership are documented
+    outcome: Guidance defines when corrections become instructions, skills, decisions, or maintained references.
+    risk: The lesson store becomes a stale archive rather than usable agent knowledge.
+    allow_agent_evidence: true
+    agent_evidence_scopes: [repository]
+    evidence:
+      - type: content_terms
+        scope: repository
+        files:
+          [
+            AGENTS.md,
+            Agents.md,
+            agents.md,
+            .ai/lessons.md,
+            .ai/memory/**/*.md,
+            .ai/instructions/**/*.md,
+            .ai/rules/**/*.md,
+            .ai/skills/**/*.md,
+            .agents/**/*.md,
+            docs/knowledge/**,
+          ]
+        terms: [curation, promote, lesson, correction, stale, owner, deduplication]
+        min_terms: 3
+        required_any_terms: [curation, promote, lesson, correction]
+        max_span_lines: 120
+        max_files_per_pattern: 50
+    remediation: Define curation cadence, promotion rules, ownership, deduplication, and stale-content handling.
+  - id: ADRB-LRN-003
+    level: 3
+    title: Knowledge curation is mechanically enforced
+    outcome: Automation detects stale, duplicate, contradictory, or unpromoted agent knowledge.
+    risk: Knowledge quality relies on memory and quietly deteriorates as the repository evolves.
+    allow_agent_evidence: true
+    agent_evidence_scopes: [repository]
+    evidence:
+      - type: ci_command
+        scope: repository
+        providers: []
+        tools: []
+        min_tools: 1
+    remediation: Automate mechanical checks and require periodic judgement over knowledge quality.
+  - id: ADRB-LRN-004
+    level: 4
+    title: Knowledge effectiveness drives improvement
+    outcome: Recurring mistakes, stale guidance, retrieval success, and correction latency are trended and improve curation.
+    risk: Knowledge grows without evidence that it changes agent behavior.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide recurring-error and knowledge-quality trends plus a recent curation improvement driven by them.
+    remediation: Measure whether promoted knowledge prevents repeated mistakes and prioritize weak guidance.

package/benchmark/v0.4/controls/observability.yaml

@@ -0,0 +1,91 @@
+dimension: observability
+controls:
+  - id: ADRB-OBS-001
+    level: 1
+    title: Agent run records are defined
+    outcome: A tracked schema or guide defines how material agent activity can be reconstructed.
+    risk: Teams cannot explain what happened after a poor change or security concern.
+    allow_agent_evidence: true
+    agent_evidence_scopes: [repository]
+    evidence:
+      - type: content_terms
+        scope: repository
+        files:
+          [
+            .ai/run-envelope.*,
+            .ai/runs/README*,
+            .agentic/run-envelope.*,
+            .agentic/runs/README*,
+            docs/observability/**,
+            docs/agentic/**,
+            evals/**,
+            evaluations/**,
+            observability/**,
+          ]
+        terms: [task, actor, harness, result, timestamp, evidence]
+        min_terms: 3
+        required_any_terms: [agent, actor, harness, agent run]
+        max_span_lines: 100
+        max_files_per_pattern: 50
+    remediation: Define a privacy-safe run record containing task, harness, result, timing, and evidence references.
+  - id: ADRB-OBS-002
+    level: 2
+    title: Run envelope and outcome taxonomy are documented
+    outcome: Run guidance covers authority, tools, budget, stop reason, verification, and outcome classification.
+    risk: Comparisons confound model, tooling, permissions, task risk, and checkout state.
+    allow_agent_evidence: true
+    agent_evidence_scopes: [repository]
+    evidence:
+      - type: content_terms
+        scope: repository
+        files:
+          [
+            .ai/run-envelope.*,
+            .ai/runs/README*,
+            .agentic/run-envelope.*,
+            .agentic/runs/README*,
+            docs/observability/**,
+            docs/agentic/**,
+            evals/**,
+            evaluations/**,
+            observability/**,
+          ]
+        terms:
+          [
+            task class,
+            tool,
+            permission,
+            budget,
+            stop reason,
+            outcome,
+            harness version,
+            verification,
+          ]
+        min_terms: 4
+        max_span_lines: 140
+        max_files_per_pattern: 50
+    remediation: Adopt a common run envelope and outcome taxonomy with privacy and retention rules.
+  - id: ADRB-OBS-003
+    level: 3
+    title: Outcome evidence is captured consistently
+    outcome: Material agent runs automatically emit retained metadata and verification outcomes.
+    risk: Only memorable successes and failures shape policy, creating selection bias.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: platform
+        prompt: Show automated run-envelope capture, completeness monitoring, retention controls, and access controls.
+    remediation: Emit structured run and outcome records automatically while minimizing sensitive content.
+  - id: ADRB-OBS-004
+    level: 4
+    title: Agent outcomes drive autonomy decisions
+    outcome: Quality, escape rate, cycle time, cost, intervention, and rollback trends change autonomy or harness policy.
+    risk: Token use or completion rate becomes a misleading proxy for engineering value and safety.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide outcome trends segmented by task and risk class and an autonomy or harness decision they changed.
+    remediation: Evaluate quality and total effort by task class and use results to narrow or expand autonomy.

package/benchmark/v0.4/controls/resilience.yaml

@@ -0,0 +1,170 @@
+dimension: resilience
+controls:
+  - id: ADRB-RES-001
+    level: 1
+    title: Changes have a documented recovery path
+    outcome: Contributors can locate actionable rollback, revert, recovery, or incident guidance.
+    risk: A harmful agent change persists because recovery is improvised under pressure.
+    allow_agent_evidence: true
+    agent_evidence_scopes: [repository]
+    evidence:
+      - type: content_terms
+        scope: repository
+        files:
+          [
+            README.md,
+            CONTRIBUTING.md,
+            AGENTS.md,
+            Agents.md,
+            agents.md,
+            .ai/AGENTS.md,
+            .ai/Agents.md,
+            .ai/constitution.md,
+            .ai/instructions/**/*.md,
+            .ai/rules/**/*.md,
+            .ai/skills/**/*.md,
+            .agents/**/*.md,
+            docs/recovery/**,
+            docs/operations/**,
+            docs/agentic/**,
+            runbooks/**,
+          ]
+        terms: [rollback, revert, recovery, incident, restore]
+        min_terms: 2
+        required_any_terms: [change, deployment, release, agent, task, pull request]
+        max_span_lines: 100
+        max_files_per_pattern: 50
+    remediation: Document how to stop work, revert code, recover affected state, and escalate an incident.
+  - id: ADRB-RES-002
+    level: 2
+    title: Failure containment and stop conditions are documented
+    outcome: Agent guidance defines mutation limits, resource budgets, retry bounds, stop triggers, escalation, and recovery ownership.
+    risk: A looping or confused agent continues spending or mutating beyond the intended scope.
+    allow_agent_evidence: true
+    agent_evidence_scopes: [repository]
+    evidence:
+      - type: content_groups
+        scope: repository
+        files:
+          [
+            AGENTS.md,
+            Agents.md,
+            agents.md,
+            SECURITY.md,
+            .ai/AGENTS.md,
+            .ai/Agents.md,
+            .ai/constitution.md,
+            .ai/instructions/**/*.md,
+            .ai/rules/**/*.md,
+            .ai/skills/**/*.md,
+            .agents/**/*.md,
+            docs/agentic/**,
+            docs/recovery/**,
+            runbooks/**,
+            templates/**,
+          ]
+        groups:
+          - id: mutation-scope
+            terms:
+              [
+                mutation limit,
+                allowed mutation,
+                allowed mutations,
+                allowed tools and mutations,
+                permitted mutation,
+                permitted mutations,
+                allowed path,
+                allowed paths,
+                allowed prefix,
+                allowed prefixes,
+                write scope,
+                out of scope,
+                outside scope,
+                edit only,
+                do not edit,
+                do not continue outside allowed paths,
+                do not continue outside the allowed paths,
+              ]
+          - id: resource-bounds
+            terms:
+              [
+                resource budget,
+                token budget,
+                time budget,
+                cost budget,
+                tool call budget,
+                tool-call budget,
+                token limit,
+                time limit,
+                cost limit,
+                tool call limit,
+                tool-call limit,
+                wall clock limit,
+                wall-clock limit,
+              ]
+          - id: retry-bounds
+            terms: [retry limit, retry budget, max retries, bounded retries, maximum attempts]
+          - id: stop-trigger
+            terms:
+              [
+                stop condition,
+                stop and ask,
+                stop and surface,
+                stop safely,
+                must stop,
+                halt work,
+                abort,
+                do not continue,
+                kill switch,
+              ]
+          - id: escalation
+            terms:
+              [
+                escalation,
+                escalate,
+                ask the coordinator,
+                ask coordinator,
+                surface to coordinator,
+                hand back,
+                ask a human,
+                escalate to a human,
+                hand back to a human,
+              ]
+          - id: recovery-ownership
+            terms:
+              [
+                recovery owner,
+                rollback owner,
+                incident owner,
+                responsible for recovery,
+                recovery responsibility,
+                on-call owner,
+              ]
+        min_groups: 6
+        max_span_lines: 120
+        max_files_per_pattern: 50
+    remediation: Add bounded scope, budget, retry, stop, escalation, and recovery requirements to task preflight.
+  - id: ADRB-RES-003
+    level: 3
+    title: Containment and recovery controls are exercised
+    outcome: Cancellation, credential revocation, rollback, timeout, and partial-failure recovery are tested.
+    risk: Recovery mechanisms exist on paper but fail when the harness behaves unexpectedly.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: organization
+        prompt: Show recent automated tests or exercises for stop, timeout, revocation, rollback, and partial failure.
+    remediation: Add failure-injection tests and periodic exercises for containment and reversible recovery.
+  - id: ADRB-RES-004
+    level: 4
+    title: Recovery outcomes drive improvement
+    outcome: Time-to-detect, time-to-stop, rollback success, impact, and recurrence trends improve containment.
+    risk: The same failure patterns repeat because recovery performance and residual impact are unknown.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide recovery outcome trends and a recent containment or rollback improvement driven by them.
+    remediation: Trend containment and recovery outcomes and feed them into autonomy and tool design.