agentic-scorecard 0.3.1 → 0.4.0

package/benchmark/v0.4/adapters/known-ci-providers.yaml

@@ -0,0 +1,57 @@
+id: known-ci-providers
+benchmark_version: 0.4.0
+extensions:
+  - control_id: ADRB-ENV-003
+    evidence_index: 0
+    ci_providers:
+      - id: github-actions
+        files: [.github/workflows/*.yml, .github/workflows/*.yaml]
+      - id: gitlab-ci
+        files: [.gitlab-ci.yml]
+      - id: azure-pipelines
+        files: [azure-pipelines.yml, azure-pipelines.yaml]
+  - control_id: ADRB-LRN-003
+    evidence_index: 0
+    ci_providers:
+      - id: github-actions
+        files: [.github/workflows/*.yml, .github/workflows/*.yaml]
+      - id: gitlab-ci
+        files: [.gitlab-ci.yml]
+      - id: azure-pipelines
+        files: [azure-pipelines.yml, azure-pipelines.yaml]
+  - control_id: ADRB-SPC-003
+    evidence_index: 0
+    ci_providers:
+      - id: github-actions
+        files: [.github/workflows/*.yml, .github/workflows/*.yaml]
+      - id: gitlab-ci
+        files: [.gitlab-ci.yml]
+      - id: azure-pipelines
+        files: [azure-pipelines.yml, azure-pipelines.yaml]
+  - control_id: ADRB-CTX-003
+    evidence_index: 0
+    ci_providers:
+      - id: github-actions
+        files: [.github/workflows/*.yml, .github/workflows/*.yaml]
+      - id: gitlab-ci
+        files: [.gitlab-ci.yml]
+      - id: azure-pipelines
+        files: [azure-pipelines.yml, azure-pipelines.yaml]
+  - control_id: ADRB-SEC-003
+    evidence_index: 0
+    ci_providers:
+      - id: github-actions
+        files: [.github/workflows/*.yml, .github/workflows/*.yaml]
+      - id: gitlab-ci
+        files: [.gitlab-ci.yml]
+      - id: azure-pipelines
+        files: [azure-pipelines.yml, azure-pipelines.yaml]
+  - control_id: ADRB-TST-003
+    evidence_index: 0
+    ci_providers:
+      - id: github-actions
+        files: [.github/workflows/*.yml, .github/workflows/*.yaml]
+      - id: gitlab-ci
+        files: [.gitlab-ci.yml]
+      - id: azure-pipelines
+        files: [azure-pipelines.yml, azure-pipelines.yaml]

package/benchmark/v0.4/adapters/known-command-wrappers.yaml

@@ -0,0 +1,37 @@
+id: known-command-wrappers
+benchmark_version: 0.4.0
+ci_invocation_grammar:
+  wrapper_options_with_values:
+    - --cache
+    - --call
+    - --chdir
+    - --chroot
+    - --close-from
+    - --command-timeout
+    - --from
+    - --group
+    - --host
+    - --node-options
+    - --package
+    - --prompt
+    - --python
+    - --user
+    - --with
+    - -c
+    - -d
+    - -g
+    - -h
+    - -p
+    - -r
+    - -t
+    - -u
+  wrappers:
+    - executable_patterns: ['^(?:bunx|npx|sudo|uvx)$']
+      command_prefixes: [[]]
+    - executable_patterns: ['^(?:uv|pipx)$']
+      command_prefixes: [[run]]
+    - executable_patterns: ['^(?:bun|npm|pnpm|yarn)$']
+      command_prefixes: [[dlx], [exec], [x]]
+    - executable_patterns: ['^python(?:3(?:\.\d+)?)?$']
+      command_prefixes: [[-m]]
+extensions: []

package/benchmark/v0.4/adapters/known-environment-tooling.yaml

@@ -0,0 +1,17 @@
+id: known-environment-tooling
+benchmark_version: 0.4.0
+extensions:
+  - control_id: ADRB-ENV-003
+    evidence_index: 0
+    ci_tools:
+      - id: locked-install
+        prohibited_arguments: [--dry-run]
+        commands:
+          - executables: [npm]
+            argument_groups: [[ci]]
+          - executables: [pnpm, yarn, bun]
+            argument_groups: [[install], [--frozen-lockfile, --immutable]]
+          - executables: [uv]
+            argument_groups: [[sync], [--frozen, --locked]]
+          - executables: [cargo]
+            argument_groups: [[fetch], [--locked]]

package/benchmark/v0.4/adapters/known-github-conventions.yaml

@@ -0,0 +1,27 @@
+id: known-github-conventions
+benchmark_version: 0.4.0
+extensions:
+  - control_id: ADRB-GOV-001
+    evidence_index: 0
+    files: [.github/pull_request_template.md, .github/PULL_REQUEST_TEMPLATE/**]
+  - control_id: ADRB-GOV-002
+    evidence_index: 0
+    patterns: [.github/CODEOWNERS, .github/OWNERS, docs/CODEOWNERS]
+  - control_id: ADRB-GOV-002
+    evidence_index: 1
+    files: [.github/pull_request_template.md]
+  - control_id: ADRB-RES-001
+    evidence_index: 0
+    files: [.github/SECURITY.md]
+  - control_id: ADRB-SEC-001
+    evidence_index: 0
+    files: [.github/SECURITY.md]
+  - control_id: ADRB-SEC-006
+    evidence_index: 0
+    files: [.github/SECURITY.md]
+  - control_id: ADRB-SPC-001
+    evidence_index: 0
+    patterns: [.github/ISSUE_TEMPLATE/**]
+  - control_id: ADRB-SPC-002
+    evidence_index: 0
+    files: [.github/ISSUE_TEMPLATE/**]

package/benchmark/v0.4/adapters/known-python-tooling.yaml

@@ -0,0 +1,15 @@
+id: known-python-tooling
+benchmark_version: 0.4.0
+extensions:
+  - control_id: ADRB-ENV-001
+    evidence_index: 0
+    files: [pyproject.toml]
+  - control_id: ADRB-ENV-001
+    evidence_index: 0
+    terms: [requires-python, uv sync, python]
+  - control_id: ADRB-TST-002
+    evidence_index: 0
+    files: [run_quality_checks.sh, scripts/**/quality*.sh, tox.ini, noxfile.py]
+  - control_id: ADRB-TST-002
+    evidence_index: 0
+    terms: [pytest, mypy, flake8, ruff, pyright, tox, nox]

package/benchmark/v0.4/adapters/known-security-tools.yaml

@@ -0,0 +1,45 @@
+id: known-security-tools
+benchmark_version: 0.4.0
+extensions:
+  - control_id: ADRB-SEC-003
+    evidence_index: 0
+    ci_tools:
+      - id: gitleaks
+        requires_final_exit_status: true
+        commands:
+          - executables: [gitleaks]
+            argument_groups: [[detect]]
+            prohibited_arguments: [--exit-code, --log-opts, --source]
+        actions: [gitleaks/gitleaks-action]
+      - id: trufflehog
+        requires_final_exit_status: true
+        commands:
+          - executables: [trufflehog]
+            argument_groups:
+              - [git, filesystem]
+              - [--fail]
+            required_argument_prefixes:
+              - [git, '.']
+              - [git, ./]
+              - [git, 'file://.']
+              - [git, 'file://./']
+              - [git, '$GITHUB_WORKSPACE']
+              - [git, '${GITHUB_WORKSPACE}']
+              - [git, '$CI_PROJECT_DIR']
+              - [git, '${CI_PROJECT_DIR}']
+              - [git, '$(Build.SourcesDirectory)']
+              - [filesystem, '.']
+              - [filesystem, ./]
+              - [filesystem, '$GITHUB_WORKSPACE']
+              - [filesystem, '${GITHUB_WORKSPACE}']
+              - [filesystem, '$CI_PROJECT_DIR']
+              - [filesystem, '${CI_PROJECT_DIR}']
+              - [filesystem, '$(Build.SourcesDirectory)']
+            prohibited_arguments: [--fail=false]
+            prohibited_argument_sequences: [[--fail, 'false']]
+      - id: git-secrets
+        requires_final_exit_status: true
+        commands:
+          - executables: [git-secrets]
+            argument_groups: [[--scan, --scan-history]]
+            max_arguments: 1

package/benchmark/v0.4/adapters/known-verification-tools.yaml

@@ -0,0 +1,65 @@
+id: known-verification-tools
+benchmark_version: 0.4.0
+extensions:
+  - control_id: ADRB-ENV-003
+    evidence_index: 0
+    ci_tools:
+      - id: tests
+        prohibited_arguments:
+          [--dry-run, --fixtures, --fixtures-per-test, --list-sessions, --markers, --no-run]
+        commands:
+          - executables: [npm, pnpm, yarn, bun, turbo, nx]
+            argument_groups:
+              - [test, test:unit, test:integration, test:e2e, unit-test, integration-test, e2e]
+          - executables: [cargo, go, dotnet, mvn, gradle, bazel]
+            repository_executables: [mvnw, gradlew]
+            argument_groups: [[test]]
+        standalone_executables: [pytest, pytest-cov, jest, vitest]
+      - id: static-analysis
+        commands:
+          - executables: [npm, pnpm, yarn, bun, turbo, nx]
+            argument_groups: [[lint, typecheck, type-check, check:types, check-types, check]]
+          - executables: [cargo]
+            argument_groups: [[clippy, check]]
+          - executables: [go]
+            argument_groups: [[vet]]
+          - executables: [mvn]
+            repository_executables: [mvnw]
+            argument_groups: [[checkstyle:check, spotbugs:check, pmd:check]]
+          - executables: [gradle]
+            repository_executables: [gradlew]
+            argument_groups: [[check]]
+          - executables: [ruff]
+            argument_groups: [[check]]
+        standalone_executables: [eslint, tsc, mypy, flake8, pyright, golangci-lint]
+  - control_id: ADRB-TST-003
+    evidence_index: 0
+    ci_tools:
+      - id: tests
+        prohibited_arguments:
+          [--dry-run, --fixtures, --fixtures-per-test, --list-sessions, --markers, --no-run]
+        commands:
+          - executables: [npm, pnpm, yarn, bun, turbo, nx]
+            argument_groups:
+              - [test, test:unit, test:integration, test:e2e, unit-test, integration-test, e2e]
+          - executables: [cargo, go, dotnet, mvn, gradle, bazel]
+            repository_executables: [mvnw, gradlew]
+            argument_groups: [[test]]
+        standalone_executables: [pytest, pytest-cov, jest, vitest]
+      - id: static-analysis
+        commands:
+          - executables: [npm, pnpm, yarn, bun, turbo, nx]
+            argument_groups: [[lint, typecheck, type-check, check:types, check-types, check]]
+          - executables: [cargo]
+            argument_groups: [[clippy, check]]
+          - executables: [go]
+            argument_groups: [[vet]]
+          - executables: [mvn]
+            repository_executables: [mvnw]
+            argument_groups: [[checkstyle:check, spotbugs:check, pmd:check]]
+          - executables: [gradle]
+            repository_executables: [gradlew]
+            argument_groups: [[check]]
+          - executables: [ruff]
+            argument_groups: [[check]]
+        standalone_executables: [eslint, tsc, mypy, flake8, pyright, golangci-lint]

package/benchmark/v0.4/agent-evidence-schema.json

@@ -0,0 +1,74 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agentic-readiness.dev/schema/v0.4/agent-evidence.json",
+  "title": "ADRB v0.4 agent-collected evidence bundle",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "benchmark_version", "target", "collector", "claims"],
+  "properties": {
+    "schema_version": { "const": "0.4.0" },
+    "benchmark_version": { "const": "0.4.0" },
+    "target": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["repository", "git_head"],
+      "properties": {
+        "repository": { "type": "string", "minLength": 1 },
+        "git_head": { "type": "string", "minLength": 1 }
+      }
+    },
+    "collector": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "version"],
+      "properties": {
+        "name": { "type": "string", "minLength": 1 },
+        "version": { "type": "string", "minLength": 1 }
+      }
+    },
+    "claims": {
+      "type": "object",
+      "propertyNames": { "pattern": "^ADRB-[A-Z]{3}-[0-9]{3}$" },
+      "additionalProperties": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["status", "scope", "summary", "references", "collected_at", "expires_at"],
+        "properties": {
+          "status": { "enum": ["met", "not_met", "unknown"] },
+          "scope": { "enum": ["repository", "platform", "organization", "outcome"] },
+          "summary": { "type": "string", "minLength": 1 },
+          "references": {
+            "type": "array",
+            "minItems": 1,
+            "items": { "type": "string", "minLength": 1 }
+          },
+          "collected_at": { "type": "string", "format": "date-time" },
+          "expires_at": { "type": "string", "format": "date-time" },
+          "error": { "type": ["string", "null"], "minLength": 1 }
+        },
+        "allOf": [
+          {
+            "if": {
+              "required": ["error"],
+              "properties": { "error": { "type": "string" } }
+            },
+            "then": { "properties": { "status": { "const": "unknown" } } }
+          },
+          {
+            "if": { "properties": { "scope": { "const": "repository" } } },
+            "then": {
+              "properties": {
+                "references": {
+                  "items": {
+                    "type": "string",
+                    "pattern": "^repo:(?!/)(?!\\.{1,2}(?:/|#|$))(?!.*\\/\\.{1,2}(?:/|#|$))(?!.*\\\\)[^#]+(?:#L[1-9][0-9]*(?:-L?[1-9][0-9]*)?)?$"
+                  }
+                }
+              }
+            }
+          }
+        ]
+      }
+    }
+  }
+}

package/benchmark/v0.4/attestation-schema.json

@@ -0,0 +1,35 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agentic-readiness.dev/schema/v0.4/attestations.json",
+  "title": "ADRB v0.4 human attestation file",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["benchmark_version", "target", "attestations"],
+  "properties": {
+    "benchmark_version": { "const": "0.4.0" },
+    "target": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["repository"],
+      "properties": {
+        "repository": { "type": "string", "minLength": 1 }
+      }
+    },
+    "attestations": {
+      "type": "object",
+      "propertyNames": { "pattern": "^ADRB-[A-Z]{3}-[0-9]{3}$" },
+      "additionalProperties": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["status", "evidence", "owner", "reviewed_at", "expires_at"],
+        "properties": {
+          "status": { "enum": ["met", "not_met", "not_applicable", "unknown"] },
+          "evidence": { "type": "string", "minLength": 1 },
+          "owner": { "type": "string", "minLength": 1 },
+          "reviewed_at": { "type": "string", "format": "date" },
+          "expires_at": { "type": "string", "format": "date" }
+        }
+      }
+    }
+  }
+}

package/benchmark/v0.4/benchmark.yaml

@@ -0,0 +1,127 @@
+id: agentic-development-readiness
+version: 0.4.0
+title: Agentic Development Readiness Benchmark
+description: >-
+  A vendor-neutral, evidence-backed assessment of the engineering harness that surrounds AI agents.
+  It measures whether a repository and its operating controls can safely support increasing levels
+  of delegated software work.
+
+maturity_levels:
+  0: Absent
+  1: Ad hoc
+  2: Documented
+  3: Enforced
+  4: Measured and improving
+
+dimensions:
+  - id: context
+    title: Context and knowledge
+    description: Agents can discover authoritative, scoped, current instructions without guesswork.
+  - id: environment
+    title: Reproducible environment
+    description: Agents can establish and verify a faithful development environment.
+  - id: specification
+    title: Specification and planning
+    description: Work starts from testable outcomes, constraints, and explicit acceptance criteria.
+  - id: tooling
+    title: Tooling and interfaces
+    description: Agent tools are discoverable, least-privileged, observable, and fail safely.
+  - id: security
+    title: Security and data governance
+    description: Credentials, data, autonomy, and untrusted content have explicit boundaries.
+  - id: testing
+    title: Verification and testing
+    description: Changes are checked at the right layers with trustworthy, reproducible signals.
+  - id: governance
+    title: Review and change governance
+    description: Human accountability, change scope, and merge authority remain explicit.
+  - id: learning
+    title: Learning and knowledge maintenance
+    description: Corrections compound into maintained instructions instead of being rediscovered.
+  - id: observability
+    title: Outcome observability
+    description: Runs and outcomes can be evaluated by task class, risk, cost, and quality.
+  - id: resilience
+    title: Failure containment and recovery
+    description: Agent failures are bounded, reversible, diagnosed, and rehearsed.
+
+readiness_profiles:
+  - id: read-only-analysis
+    title: Read-only analysis
+    description: Agent may inspect approved repository content and return advice, without mutations.
+    floors:
+      context: 1
+      environment: 0
+      specification: 0
+      tooling: 0
+      security: 1
+      testing: 0
+      governance: 0
+      learning: 0
+      observability: 0
+      resilience: 0
+  - id: planning
+    title: Planning and specification
+    description: Agent may draft plans, specifications, and proposed changes for human review.
+    floors:
+      context: 2
+      environment: 0
+      specification: 1
+      tooling: 1
+      security: 1
+      testing: 0
+      governance: 1
+      learning: 0
+      observability: 0
+      resilience: 0
+  - id: local-implementation
+    title: Local implementation
+    description: Agent may edit an isolated checkout and run approved local verification.
+    floors:
+      context: 2
+      environment: 2
+      specification: 2
+      tooling: 2
+      security: 2
+      testing: 2
+      governance: 1
+      learning: 1
+      observability: 1
+      resilience: 1
+  - id: pr-creation
+    title: Pull request creation
+    description: Agent may create a branch and propose a pull request; a human retains merge authority.
+    floors:
+      context: 2
+      environment: 2
+      specification: 2
+      tooling: 2
+      security: 2
+      testing: 2
+      governance: 2
+      learning: 1
+      observability: 1
+      resilience: 2
+  - id: limited-autonomous-maintenance
+    title: Limited autonomous maintenance
+    description: >-
+      Agent may complete pre-approved, low-risk maintenance within strict scope, budget, and stop
+      conditions. This profile never grants production deployment authority.
+    floors:
+      context: 3
+      environment: 3
+      specification: 3
+      tooling: 3
+      security: 3
+      testing: 3
+      governance: 3
+      learning: 3
+      observability: 3
+      resilience: 3
+
+scoring:
+  dimension_method: consecutive-levels
+  overall_method: sum
+  maximum_score: 40
+  readiness_method: non-compensating-floors
+  attestation_counts_as_verified: false

package/benchmark/v0.4/calibration-matrix.md

@@ -0,0 +1,60 @@
+# v0.4 calibration matrix
+
+This matrix records the versioned expected outcomes exercised by the v0.4 conformance fixtures.
+PR #11 review is the independent acceptance gate for these expectations. The executable assertions
+live in `tests/v04-calibration.test.ts`; this document is the public, human-readable index.
+
+| Case                                                                                                                                 | Surface                                          | Expected outcome                                               |
+| ------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------ | -------------------------------------------------------------- |
+| Mature harness                                                                                                                       | Node, GitHub Actions                             | 30/40, repository 23/23, limited-autonomous-maintenance passes |
+| Guidance and verification comments only                                                                                              | GitHub Actions plus Jenkins comments             | CTX-003 and TST-003 do not pass                                |
+| Missing or display-only package tasks                                                                                                | Node package scripts                             | CTX-003 and TST-003 do not pass                                |
+| Missing package tasks behind supported wrappers                                                                                      | Node, GitHub Actions                             | TST-003 does not pass                                          |
+| Package/workspace context switch; nested package-exec tools                                                                          | Node, GitHub Actions                             | Context switch fails closed; nested tools may pass             |
+| Executed no-op guidance-validation script                                                                                            | Node, GitHub Actions                             | CTX-003 does not pass                                          |
+| Executed guidance validation, tests, and typecheck                                                                                   | Node, GitHub Actions                             | CTX-003 and TST-003 pass                                       |
+| Locked install, tests, static analysis, traceability, and curation                                                                   | Node, GitHub Actions                             | ENV-003, SPC-003, and LRN-003 pass                             |
+| Frozen install with pnpm, Yarn, or Bun                                                                                               | Node, GitHub Actions                             | ENV-003 passes                                                 |
+| Keyword-bearing no-op environment, traceability, or curation files                                                                   | Node, GitHub Actions                             | ENV-003, SPC-003, and LRN-003 do not pass                      |
+| Specification heading check without implementation/test bindings                                                                     | Node, GitHub Actions                             | SPC-003 does not pass                                          |
+| Python tests and static analysis                                                                                                     | `uv run pytest` and `uv run mypy`, GitLab CI     | TST-003 passes                                                 |
+| Rust tests and static analysis                                                                                                       | `cargo test` and `cargo clippy`, Azure Pipelines | TST-003 passes                                                 |
+| Tests and static analysis split across CI files                                                                                      | Separate GitHub workflows                        | TST-003 passes                                                 |
+| Tests and static analysis in one fail-fast multiline step                                                                            | GitHub Actions                                   | TST-003 passes                                                 |
+| Multiline verification without a provider fail-fast guarantee                                                                        | Azure Pipelines                                  | TST-003 does not pass                                          |
+| Test discovery or session-listing commands plus static analysis                                                                      | Python, GitHub Actions                           | TST-003 does not pass                                          |
+| Go or .NET test-listing modes plus static analysis                                                                                   | Multiple runtimes, GitHub Actions                | TST-003 does not pass                                          |
+| Pytest local-variable display plus static analysis                                                                                   | Python, GitHub Actions                           | TST-003 passes                                                 |
+| Tracked Maven/Gradle root wrappers versus missing wrapper                                                                            | Java, GitHub Actions                             | Tracked wrappers pass; missing wrapper does not                |
+| Monorepo tests and static analysis                                                                                                   | Root package tasks bound to Turbo commands       | TST-003 passes                                                 |
+| Invalid cross-tool command aliases                                                                                                   | GitHub Actions                                   | TST-003 command classes remain unmatched                       |
+| Path-qualified package-manager impersonator or command after unconditional exit                                                      | Node, GitHub Actions                             | Mechanical CI controls do not pass                             |
+| Executed secret scanner                                                                                                              | GitHub Actions, GitLab CI, Azure Pipelines       | SEC-003 repository alternative passes                          |
+| git-secrets default scan versus unrelated explicit target                                                                            | GitHub Actions                                   | Default passes; unrelated target does not                      |
+| Scanner comments, display modes, disabled/non-blocking jobs, neutralized scanner exits, invalid actions, or non-integration triggers | Multiple providers                               | SEC-003 repository alternative does not pass                   |
+| Explicit unsupported command shell                                                                                                   | GitHub Actions                                   | SEC-003 repository alternative does not pass                   |
+| Disjunction of supported integration events                                                                                          | GitHub Actions                                   | SEC-003 repository alternative passes                          |
+| Unresolved GitLab inheritance, exclusions, or non-blocking defaults                                                                  | GitLab CI                                        | SEC-003 repository alternative does not pass                   |
+| Malformed GitLab workflow or job rules                                                                                               | GitLab CI                                        | SEC-003 repository alternative does not pass                   |
+| Job-level merge-request rule under a workflow-level denial                                                                           | GitLab CI                                        | SEC-003 repository alternative does not pass                   |
+| Nested CI fragments or Azure root command fields                                                                                     | Multiple providers                               | SEC-003 repository alternative does not pass                   |
+| Conditional checkout with unsupported target effect; non-root setup hook or working directory                                        | Multiple providers                               | Repository-bound CI evidence fails closed                      |
+| Azure stage with direct steps                                                                                                        | Azure Pipelines                                  | SEC-003 repository alternative does not pass                   |
+| Host-native secret scanning claim                                                                                                    | Target-bound platform evidence                   | SEC-003 may pass; SEC-007 remains unresolved                   |
+| Keyword-stuffed or distant guidance                                                                                                  | Repository documents and generated run output    | Mechanical Level 3 controls do not pass                        |
+| Comment-only validation-script structure                                                                                             | Executed tracked validator                       | CTX-003 does not pass                                          |
+| Required validator patterns split across unreachable functions                                                                       | Executed tracked validator                       | CTX-003 does not pass                                          |
+| Explicit ownership map plus human approval and merge authority                                                                       | Conventional ownership formats                   | GOV-002 passes                                                 |
+| Placeholder/inactive owner, vague prose, negated authority, or missing authority                                                     | Conventional ownership formats                   | GOV-002 does not pass                                          |
+| AI/bot-qualified maintainer approval or merge authority                                                                              | Agent guidance                                   | GOV-002 does not pass                                          |
+| Component table in `OWNERS.md`; contact under a former-maintainers heading                                                           | Conventional ownership formats                   | Active mapping passes; former-only contact does not            |
+| Bare component name in a typed ownership table                                                                                       | Conventional ownership formats                   | Accountable assignment passes                                  |
+| Complete co-located containment components                                                                                           | Agent guidance                                   | RES-002 passes                                                 |
+| Compound “do not continue outside allowed paths” containment rule                                                                    | Agent guidance                                   | Mutation scope and stop trigger both match                     |
+| Resource budget without retry bounds, or retry bounds without a resource budget                                                      | Agent guidance                                   | RES-002 does not pass                                          |
+| Partial, negated, or absent containment components                                                                                   | Agent guidance                                   | RES-002 does not pass and reports matched/missing groups       |
+
+Jenkins is intentionally a negative calibration in v0.4: the offline collector does not claim that
+a Jenkinsfile is integration-triggered without a supported structural provider parser. Such evidence
+remains unresolved or may be supplied as eligible, source-backed agent evidence. Scores from v0.3
+and v0.4 are not directly comparable.

package/benchmark/v0.4/controls/context.yaml

@@ -0,0 +1,75 @@
+dimension: context
+controls:
+  - id: ADRB-CTX-001
+    level: 1
+    title: Agent guidance is discoverable
+    outcome: A contributor or agent can locate the repository's authoritative operating guidance.
+    risk: Agents infer repository rules from generic practice or stale neighboring files.
+    evidence:
+      - type: path_any
+        scope: repository
+        patterns:
+          [
+            AGENTS.md,
+            Agents.md,
+            agents.md,
+            .ai/AGENTS.md,
+            .ai/Agents.md,
+            .ai/constitution.md,
+            .ai/instructions/**,
+            .ai/rules/**,
+            .agents/**,
+          ]
+    remediation: Add a tracked, non-empty agent entry point that names the authoritative guidance.
+  - id: ADRB-CTX-002
+    level: 2
+    title: Guidance scope and precedence are explicit
+    outcome: Instructions state their scope, precedence, important constraints, and architecture boundaries.
+    risk: Conflicting or nested guidance produces plausible changes that violate local invariants.
+    allow_agent_evidence: true
+    agent_evidence_scopes: [repository]
+    evidence:
+      - type: content_terms
+        scope: repository
+        files:
+          [
+            AGENTS.md,
+            Agents.md,
+            agents.md,
+            .ai/AGENTS.md,
+            .ai/Agents.md,
+            .ai/constitution.md,
+            .ai/instructions/**/*.md,
+            .ai/rules/**/*.md,
+            .ai/skills/**/*.md,
+            .agents/**/*.md,
+          ]
+        terms: [scope, precedence, architecture, must, constraint]
+        min_terms: 2
+        max_span_lines: 120
+        max_files_per_pattern: 50
+    remediation: Document instruction scope, precedence, non-negotiable constraints, and where deeper guidance lives.
+  - id: ADRB-CTX-003
+    level: 3
+    title: Agent guidance integrity is mechanically checked
+    outcome: Automated validation detects missing, broken, stale, or structurally invalid agent guidance.
+    risk: Critical instructions silently drift while agents continue treating them as authoritative.
+    allow_agent_evidence: true
+    agent_evidence_scopes: [repository]
+    evidence:
+      - type: ci_command
+        scope: repository
+        min_tools: 1
+    remediation: Add a CI check for required agent files, links, ownership, and structural invariants.
+  - id: ADRB-CTX-004
+    level: 4
+    title: Guidance effectiveness drives improvement
+    outcome: Retrieval failures, repeated instruction errors, and correction latency are trended and improve guidance.
+    risk: Documentation volume grows without evidence that agents find and apply the right context.
+    allow_attestation: true
+    allow_agent_evidence: true
+    evidence:
+      - type: manual
+        scope: outcome
+        prompt: Provide time-series guidance-quality outcomes and a recent instruction change driven by them.
+    remediation: Measure context retrieval and recurring instruction failures, then update guidance from the results.