agentme 0.28.1 → 0.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/.filedist-package.yml +2 -2
  2. package/.xdrs/agentme/edrs/application/{003-javascript-project-tooling.md → 101-javascript-project-tooling.md} +9 -9
  3. package/.xdrs/agentme/edrs/application/{010-golang-project-tooling.md → 102-golang-project-tooling.md} +7 -7
  4. package/.xdrs/agentme/edrs/application/{014-python-project-tooling.md → 103-python-project-tooling.md} +12 -12
  5. package/.xdrs/agentme/edrs/application/{015-cli-tool-standards.md → 104-cli-tool-standards.md} +13 -13
  6. package/.xdrs/agentme/edrs/application/{002-coding-best-practices.md → 121-coding-best-practices.md} +3 -3
  7. package/.xdrs/agentme/edrs/application/{004-unit-test-requirements.md → 122-unit-test-requirements.md} +2 -2
  8. package/.xdrs/agentme/edrs/application/{009-error-handling.md → 123-error-handling.md} +2 -2
  9. package/.xdrs/agentme/edrs/application/{022-secrets-management.md → 124-secrets-management.md} +5 -5
  10. package/.xdrs/agentme/edrs/application/{023-coding-abstraction-practices.md → 125-coding-abstraction-practices.md} +3 -3
  11. package/.xdrs/agentme/edrs/application/{026-pragmatic-hexagonal-architecture.md → 126-pragmatic-hexagonal-architecture.md} +9 -9
  12. package/.xdrs/agentme/edrs/application/127-external-system-adapter-skills.md +93 -0
  13. package/.xdrs/agentme/edrs/application/{040-ai-llm-development-standards.md → 141-ai-llm-development-standards.md} +14 -14
  14. package/.xdrs/agentme/edrs/application/{041-ai-agents-development-standards.md → 142-ai-agents-development-standards.md} +11 -11
  15. package/.xdrs/agentme/edrs/application/{042-ai-agents-quality-standards.md → 143-ai-agents-quality-standards.md} +16 -16
  16. package/.xdrs/agentme/edrs/application/{043-ai-workflow-development-standards.md → 144-ai-workflow-development-standards.md} +28 -28
  17. package/.xdrs/agentme/edrs/application/{044-ai-workflow-naming-conventions.md → 145-ai-workflow-naming-conventions.md} +9 -9
  18. package/.xdrs/agentme/edrs/application/{045-ai-agent-xdrs-knowledge-layer.md → 146-ai-agent-xdrs-knowledge-layer.md} +7 -7
  19. package/.xdrs/agentme/edrs/application/{051-ai-eval-core-standards.md → 151-ai-eval-standards.md} +17 -17
  20. package/.xdrs/agentme/edrs/application/{052-ai-test-types-taxonomy.md → 152-ai-test-types-taxonomy.md} +27 -27
  21. package/.xdrs/agentme/edrs/application/{053-ai-eval-script.md → 153-ai-eval-script.md} +18 -18
  22. package/.xdrs/agentme/edrs/application/{054-ai-eval-report-format.md → 154-ai-eval-report-format.md} +6 -6
  23. package/.xdrs/agentme/edrs/application/{055-ai-eval-repeatability.md → 155-ai-eval-repeatability.md} +14 -14
  24. package/.xdrs/agentme/edrs/application/skills/{004-select-relevant-xdrs → 010-select-relevant-xdrs}/SKILL.md +10 -10
  25. package/.xdrs/agentme/edrs/application/skills/{001-create-javascript-project → 050-create-javascript-project}/SKILL.md +5 -5
  26. package/.xdrs/agentme/edrs/application/skills/{003-create-golang-project → 051-create-golang-project}/SKILL.md +4 -4
  27. package/.xdrs/agentme/edrs/application/skills/{005-create-python-project → 052-create-python-project}/SKILL.md +6 -6
  28. package/.xdrs/agentme/edrs/application/skills/200-run-skill-tests/SKILL.md +113 -0
  29. package/.xdrs/agentme/edrs/application/skills/200-run-skill-tests/SKILL.test.md +47 -0
  30. package/.xdrs/agentme/edrs/data/{050-ml-dataset-structure.md → 201-ml-dataset-structure.md} +4 -4
  31. package/.xdrs/agentme/edrs/governance/{007-project-quality-standards.md → 501-project-quality-standards.md} +16 -16
  32. package/.xdrs/agentme/edrs/governance/{013-contributing-guide-requirements.md → 502-contributing-guide-requirements.md} +4 -4
  33. package/.xdrs/agentme/edrs/index.md +33 -31
  34. package/.xdrs/agentme/edrs/operations/{011-service-health-check-endpoint.md → 401-service-health-check-endpoint.md} +2 -2
  35. package/.xdrs/agentme/edrs/platform/{005-monorepo-structure.md → 301-monorepo-structure.md} +5 -5
  36. package/.xdrs/agentme/edrs/platform/{006-github-pipelines.md → 302-github-pipelines.md} +3 -3
  37. package/.xdrs/agentme/edrs/platform/{008-common-targets.md → 303-common-targets.md} +8 -8
  38. package/.xdrs/agentme/edrs/platform/{017-tool-execution-and-scripting.md → 304-tool-execution-and-scripting.md} +10 -10
  39. package/.xdrs/agentme/edrs/platform/{027-environment-variable-configuration.md → 305-environment-variable-configuration.md} +14 -14
  40. package/.xdrs/agentme/edrs/platform/skills/{002-monorepo-setup → 053-monorepo-setup}/SKILL.md +4 -4
  41. package/.xdrs/agentme/edrs/principles/012-continuous-xdr-enrichment.md +2 -2
  42. package/.xdrs/agentme/edrs/principles/016-cross-language-module-structure.md +4 -4
  43. package/.xdrs/agentme/edrs/principles/017-skill-testing.md +77 -0
  44. package/.xdrs/agentme/edrs/principles/articles/001-continuous-xdr-improvement.md +1 -1
  45. package/package.json +2 -2
@@ -1,11 +1,11 @@
1
1
  ---
2
- name: agentme-edr-policy-052-ai-test-types-taxonomy
3
- description: Names AI-application test types grouped as "Safety & adversarial", "Responsible AI", "Quality eval", "Prompt/LLM", and "Code-level" with their objective, mocking constraint, and relevance, and defines the shared "golden dataset" entry envelope that agentme-edr-051's eval tooling filters by test_types. Use when deciding which AI test types to implement or when authoring a golden dataset entry.
2
+ name: agentme-edr-policy-152-ai-test-types-taxonomy
3
+ description: Names AI-application test types grouped as "Safety & adversarial", "Responsible AI", "Quality eval", "Prompt/LLM", and "Code-level" with their objective, mocking constraint, and relevance, and defines the shared "golden dataset" entry envelope that agentme-edr-151's eval tooling filters by test_types. Use when deciding which AI test types to implement or when authoring a golden dataset entry.
4
4
  apply-to: AI projects (LLM, Agent, or Workflow tier) implementing AI-specific test types beyond generic code-level unit/integration tests
5
5
  valid-from: 2026-07-05
6
6
  ---
7
7
 
8
- # agentme-edr-policy-052: AI test types taxonomy
8
+ # agentme-edr-policy-152: AI test types taxonomy
9
9
 
10
10
  ## Context and Problem Statement
11
11
 
@@ -13,7 +13,7 @@ AI components need test types beyond generic unit/integration tests (safety, fai
13
13
 
14
14
  ## Decision Outcome
15
15
 
16
- **Adopt a named taxonomy of AI test types plus a shared "golden dataset" entry envelope that agentme-edr-051's eval tooling filters by `test_types`.**
16
+ **Adopt a named taxonomy of AI test types plus a shared "golden dataset" entry envelope that agentme-edr-151's eval tooling filters by `test_types`.**
17
17
 
18
18
  Each test type is named with its group, objective, mocking constraint, applicability, and relevance; every golden dataset entry is labeled with the test types it applies to.
19
19
 
@@ -21,7 +21,7 @@ Each test type is named with its group, objective, mocking constraint, applicabi
21
21
 
22
22
  #### 01-golden-dataset-concept
23
23
 
24
- Projects MUST use a golden dataset to test AI components. A **golden dataset** comprises all eval case entries used to test an AI component (LLM, Agent, or Workflow tier); each entry is labeled with the `test_types` (rule `04`) it applies to. It is the dataset consumed by [agentme-edr-053](053-ai-eval-script.md) evals and stored as one JSON file per entry per [agentme-edr-050](../data/050-ml-dataset-structure.md) rule `04`, at `evals/<component>/eval-<name>/golden_dataset/`.
24
+ Projects MUST use a golden dataset to test AI components. A **golden dataset** comprises all eval case entries used to test an AI component (LLM, Agent, or Workflow tier); each entry is labeled with the `test_types` (rule `04`) it applies to. It is the dataset consumed by [agentme-edr-153](153-ai-eval-script.md) evals and stored as one JSON file per entry per [agentme-edr-201](../data/201-ml-dataset-structure.md) rule `04`, at `evals/<component>/eval-<name>/golden_dataset/`.
25
25
 
26
26
  #### 02-golden-dataset-entry-envelope
27
27
 
@@ -42,8 +42,8 @@ Every golden dataset entry (a JSON file in `golden_dataset/data/`) MUST have thi
42
42
  - `test_types` — array, values MUST come from rule `04`'s enum, MUST contain at least one value. An entry MAY carry more than one value additively (e.g. `["functional", "smoke", "human"]`) — no test type excludes another.
43
43
  - `input` — for Prompt-tier components, a raw prompt string or the prompt template's input parameters object; for Agent/Workflow-tier components, the input attributes object passed to the component.
44
44
  - `expected_output` — the fields used to score the entry under each of its automated `test_types`: output attributes for an LLM-as-judge rubric, a target for vector-similarity scoring, or exact attribute values for strict comparison. When `human` is one of the entry's `test_types`, `expected_output` MUST additionally include a `human_test` string field with manual-verification instructions (e.g. `"check for ethical issues, verify record change in system X"`) — this supplements, and MUST NOT replace, the entry's automated scoring fields.
45
- - `mock_fixtures` — optional object; keys identify the adapter or external system to mock (SHOULD match the connector folder name under `adapters/connectors/<name>` for readability, though not enforced), values are any valid JSON interpreted by the mock implementation. When present, eval.py MUST configure each named mock adapter with its fixture data BEFORE invoking the component for that entry; each entry MUST use fresh mock instances to prevent state from bleeding across entries. `mock_fixtures` applies to all `test_types` including `human` — the component is still invoked for human entries to capture `actual_output`. `mock_fixtures` MUST NOT include keys for LLM adapters: all golden dataset test types are rated `mocks disallowed for LLM calls` (rule `03`), so the LLM call MUST be real; LLM provider mocking belongs exclusively to unit tests via [agentme-edr-040](040-ai-llm-development-standards.md) rule `04`. See [agentme-edr-026](026-pragmatic-hexagonal-architecture.md) rule `10` for the `_mock` file naming and placement convention.
46
- - The dataset's `dataset.schema.json` MUST require `test_types`, `input`, and `expected_output`, and SHOULD declare `mock_fixtures` as optional (`"type": "object", "additionalProperties": {}`), per [agentme-edr-050](../data/050-ml-dataset-structure.md) rule `04`.
45
+ - `mock_fixtures` — optional object; keys identify the adapter or external system to mock (SHOULD match the connector folder name under `adapters/connectors/<name>` for readability, though not enforced), values are any valid JSON interpreted by the mock implementation. When present, eval.py MUST configure each named mock adapter with its fixture data BEFORE invoking the component for that entry; each entry MUST use fresh mock instances to prevent state from bleeding across entries. `mock_fixtures` applies to all `test_types` including `human` — the component is still invoked for human entries to capture `actual_output`. `mock_fixtures` MUST NOT include keys for LLM adapters: all golden dataset test types are rated `mocks disallowed for LLM calls` (rule `03`), so the LLM call MUST be real; LLM provider mocking belongs exclusively to unit tests via [agentme-edr-141](141-ai-llm-development-standards.md) rule `04`. See [agentme-edr-126](126-pragmatic-hexagonal-architecture.md) rule `10` for the `_mock` file naming and placement convention.
46
+ - The dataset's `dataset.schema.json` MUST require `test_types`, `input`, and `expected_output`, and SHOULD declare `mock_fixtures` as optional (`"type": "object", "additionalProperties": {}`), per [agentme-edr-201](../data/201-ml-dataset-structure.md) rule `04`.
47
47
 
48
48
  #### 03-mocks-allowed-values
49
49
 
@@ -51,13 +51,13 @@ The taxonomy in rule `05` rates each test type using one of three values under t
51
51
 
52
52
  | Value | Meaning |
53
53
  |---|---|
54
- | `mocks allowed` | Fully offline; fakes may replace every dependency including the LLM (e.g. `FakeListChatModel` per [agentme-edr-040](040-ai-llm-development-standards.md) rule `04`). Used only for code-level unit tests. |
54
+ | `mocks allowed` | Fully offline; fakes may replace every dependency including the LLM (e.g. `FakeListChatModel` per [agentme-edr-141](141-ai-llm-development-standards.md) rule `04`). Used only for code-level unit tests. |
55
55
  | `mocks disallowed` | No mocking of any dependency — all real external systems required. Used for integration tests. |
56
56
  | `mocks disallowed for LLM calls` | **The LLM call MUST be real; all other external dependencies (databases, APIs, external services) MAY and SHOULD be mocked via `mock_fixtures`.** `mock_fixtures` keys MUST NOT reference LLM adapters. This is the standard constraint for every golden-dataset eval test type. See rule `08` for rationale. |
57
57
 
58
58
  #### 04-test-types-enum
59
59
 
60
- A golden dataset entry's `test_types` array MUST only use these values: `safety`, `adversarial`, `fairness`, `bias`, `robustness`, `explainability`, `groundedness`, `functional`, `prompt`, `smoke`, `human`, `repeatability`. These correspond to the dataset-driven rows of rule `05`. **Unit test** and **Integration test** (the two Code-level rows) are NOT part of this enum — they have no golden dataset entries and remain governed entirely by [agentme-edr-004](004-unit-test-requirements.md) and [agentme-edr-007](../governance/007-project-quality-standards.md) rule `08`.
60
+ A golden dataset entry's `test_types` array MUST only use these values: `safety`, `adversarial`, `fairness`, `bias`, `robustness`, `explainability`, `groundedness`, `functional`, `prompt`, `smoke`, `human`, `repeatability`. These correspond to the dataset-driven rows of rule `05`. **Unit test** and **Integration test** (the two Code-level rows) are NOT part of this enum — they have no golden dataset entries and remain governed entirely by [agentme-edr-122](122-unit-test-requirements.md) and [agentme-edr-501](../governance/501-project-quality-standards.md) rule `08`.
61
61
 
62
62
  #### 05-test-type-taxonomy
63
63
 
@@ -74,43 +74,43 @@ Test types MUST be selected from this taxonomy. Each test type is named with its
74
74
  | Groundedness (RAG) eval | `groundedness` | Quality eval | Verify the answer is supported by retrieved context | mocks disallowed for LLM calls | System uses retrieval-augmented generation | Avoids confidently-wrong answers reaching customers | Pinpoints retrieval/prompt bugs | 4 |
75
75
  | Repeatability test | `repeatability` | Quality eval | Verify output stability/variance across N repeated invocations of the same input under fixed configuration | mocks disallowed for LLM calls | Non-deterministic components (temperature > 0, agentic tool-selection loops, sampling-based decoding) used in decision-critical or user-facing flows | Protects against silently flaky behavior reaching production; supports consistency SLAs | Detects prompt/agent designs too sensitive to sampling noise; informs temperature/seed tuning | 3 |
76
76
  | Human evaluation | `human` | Quality eval | Manually verify aspects automated scoring can't (ethics, side effects, external state) | mocks disallowed for LLM calls | Before major releases; periodic spot-check | Defensible, human-reviewed sign-off | Catches what automated metrics miss | 3 |
77
- | Functional eval (golden-dataset accuracy / LLM-as-judge) | `functional` | Quality eval | Measure output correctness against the golden dataset | mocks disallowed for LLM calls | Required before every Workflow release ([agentme-edr-007](../governance/007-project-quality-standards.md) rule `09`); advised elsewhere | Auditable evidence of business correctness before release | Detects regressions from model/provider/prompt changes | 5 |
77
+ | Functional eval (golden-dataset accuracy / LLM-as-judge) | `functional` | Quality eval | Measure output correctness against the golden dataset | mocks disallowed for LLM calls | Required before every Workflow release ([agentme-edr-501](../governance/501-project-quality-standards.md) rule `09`); advised elsewhere | Auditable evidence of business correctness before release | Detects regressions from model/provider/prompt changes | 5 |
78
78
  | Smoke test | `smoke` | Quality eval | Fast pass/fail check on a small, critical subset before running fuller suites | mocks disallowed for LLM calls | Every commit/PR, before functional/responsible-AI evals run | Cheap early warning before slower evals run | Fast, cheap feedback loop | 4 |
79
79
  | Prompt regression test | `prompt` | Prompt/LLM | Detect behavior change when a prompt or model version changes | mocks disallowed for LLM calls | Whenever a prompt template or model version changes | Prevents shipping a worse experience via a "small" tweak | Fast check on every prompt edit | 3 |
80
80
  | Integration test | n/a — code-level only (see rule `04`) | Code-level | Verify real interaction with external systems | mocks disallowed | Component depends on external systems | Reduces production outages from integration mismatches | Catches wiring bugs unit tests can't see | 2 |
81
- | Unit test (offline, mocked) | n/a — code-level only (see rule `04`) | Code-level | Verify deterministic logic in isolation, offline | mocks allowed | Required for Workflow tier every commit ([agentme-edr-007](../governance/007-project-quality-standards.md) rule `09`) | Lowest-cost point to catch defects | Fastest, fully offline feedback on every commit | 5 |
81
+ | Unit test (offline, mocked) | n/a — code-level only (see rule `04`) | Code-level | Verify deterministic logic in isolation, offline | mocks allowed | Required for Workflow tier every commit ([agentme-edr-501](../governance/501-project-quality-standards.md) rule `09`) | Lowest-cost point to catch defects | Fastest, fully offline feedback on every commit | 5 |
82
82
 
83
83
  #### 06-priority-and-relevance-are-descriptive-only
84
84
 
85
- Priority, Relevance, and When to Apply in rule `05` are guidance for prioritization conversations — they MUST NOT be treated as mandating which test types a project must implement, nor their thresholds. [agentme-edr-007](../governance/007-project-quality-standards.md) rule `09` remains the only tier-level testing requirement in force (Workflow unit tests + functional evals). Once a project chooses to implement and threshold a test type, [agentme-edr-053](053-ai-eval-script.md) rule `01`'s failing-threshold behavior applies uniformly, regardless of this table's priority rating — a project may enforce fairness at 70% and functional at 90%, or skip fairness entirely; that choice is a project/business decision, not one this Policy makes.
85
+ Priority, Relevance, and When to Apply in rule `05` are guidance for prioritization conversations — they MUST NOT be treated as mandating which test types a project must implement, nor their thresholds. [agentme-edr-501](../governance/501-project-quality-standards.md) rule `09` remains the only tier-level testing requirement in force (Workflow unit tests + functional evals). Once a project chooses to implement and threshold a test type, [agentme-edr-153](153-ai-eval-script.md) rule `01`'s failing-threshold behavior applies uniformly, regardless of this table's priority rating — a project may enforce fairness at 70% and functional at 90%, or skip fairness entirely; that choice is a project/business decision, not one this Policy makes.
86
86
 
87
87
  #### 07-smoke-is-distinct-from-test-smoke
88
88
 
89
- The `smoke` test type (surfaced as the `eval-smoke` Makefile target, a fast subset of the golden-dataset functional eval) is a different concept from [agentme-edr-008](../platform/008-common-targets.md)'s existing `test-smoke` target (a fast subset of code-level tests). Both MAY exist in the same project; teams MUST NOT conflate them.
89
+ The `smoke` test type (surfaced as the `eval-smoke` Makefile target, a fast subset of the golden-dataset functional eval) is a different concept from [agentme-edr-303](../platform/303-common-targets.md)'s existing `test-smoke` target (a fast subset of code-level tests). Both MAY exist in the same project; teams MUST NOT conflate them.
90
90
 
91
91
  #### 08-eval-mocking-constraint
92
92
 
93
- For every golden-dataset eval test type: **the LLM call MUST be real; all other external dependencies MUST be mocked via `mock_fixtures`.** This applies equally to `human` entries — the component is invoked to capture `actual_output` and external dependencies must be deterministic. `mock_fixtures` keys MUST NOT reference LLM adapters. Code-level unit tests are the correct place for fully offline, LLM-mocked testing (see [agentme-edr-040](040-ai-llm-development-standards.md) rule `04`).
93
+ For every golden-dataset eval test type: **the LLM call MUST be real; all other external dependencies MUST be mocked via `mock_fixtures`.** This applies equally to `human` entries — the component is invoked to capture `actual_output` and external dependencies must be deterministic. `mock_fixtures` keys MUST NOT reference LLM adapters. Code-level unit tests are the correct place for fully offline, LLM-mocked testing (see [agentme-edr-141](141-ai-llm-development-standards.md) rule `04`).
94
94
 
95
95
  #### 09-repeatability-vs-reproducibility
96
96
 
97
97
  | Property | Definition | What varies | Measured by |
98
98
  |---|---|---|---|
99
- | **Repeatability** | Output stability across N invocations at non-zero temperature | Model sampling variance | `repeatability` test type per [agentme-edr-055](055-ai-eval-repeatability.md) |
100
- | **Reproducibility** | Deterministic output at temperature = 0 with fixed seed | Nothing — any variance is a config bug | Not a golden-dataset type; verified via config, documented in [agentme-edr-027](../platform/027-environment-variable-configuration.md) |
99
+ | **Repeatability** | Output stability across N invocations at non-zero temperature | Model sampling variance | `repeatability` test type per [agentme-edr-155](155-ai-eval-repeatability.md) |
100
+ | **Reproducibility** | Deterministic output at temperature = 0 with fixed seed | Nothing — any variance is a config bug | Not a golden-dataset type; verified via config, documented in [agentme-edr-305](../platform/305-environment-variable-configuration.md) |
101
101
 
102
102
  A component may satisfy reproducibility (temperature = 0) yet still need repeatability tests for its production configuration (temperature > 0). The `repeatability` test type MUST NOT be applied to components with intentionally diverse output (brainstorming, creative generation) — variance is correct behavior there.
103
103
 
104
104
  ## References
105
105
 
106
- - [agentme-edr-050](../data/050-ml-dataset-structure.md) — Golden dataset file layout, per-entry JSON format, `$schema` pointer, and schema-lint validation
107
- - [agentme-edr-051](051-ai-eval-core-standards.md) — AI eval core standards: eval folder structure and Makefile targets (rule `01`); LLM-as-judge binary scoring contract (rule `02`)
108
- - [agentme-edr-053](053-ai-eval-script.md) — AI eval script: `--type` filtering, entry-first loop, `mock_fixtures`, threshold enforcement, and MLflow conventions
109
- - [agentme-edr-054](054-ai-eval-report-format.md) — AI eval report format: per-type `report-<type>.md` that consumes this taxonomy's test types
110
- - [agentme-edr-055](055-ai-eval-repeatability.md) — AI eval repeatability: `REPEAT_COUNT` loop exception, scoring constants (`EVAL_MIN_ACCURACY_REPEATABILITY`, `REPEAT_SEMANTIC_SIMILARITY_SCORE`), scoring methods, `repeatability_accuracy` MLflow metric, report shape, and run cadence
111
- - [agentme-edr-026](026-pragmatic-hexagonal-architecture.md) — Rule `10`: `_mock` file naming and placement convention for mock adapters referenced by `mock_fixtures`
112
- - [agentme-edr-007](../governance/007-project-quality-standards.md) — Rule `09` tier-level testing requirements (the only mandated AI testing baseline)
113
- - [agentme-edr-008](../platform/008-common-targets.md) — Rule `03` `eval-<qualifier>` Makefile convention; rule `03`'s `test-smoke` (distinguished in rule `07`)
114
- - [agentme-edr-027](../platform/027-environment-variable-configuration.md) — Environment-configuration conventions referenced in rule `09`'s reproducibility disambiguation
115
- - [agentme-edr-040](040-ai-llm-development-standards.md) — LLM tier definition and mocking utilities referenced by the `mocks allowed` value
116
- - [agentme-edr-004](004-unit-test-requirements.md) — Unit test requirements underlying the Code-level rows
106
+ - [agentme-edr-201](../data/201-ml-dataset-structure.md) — Golden dataset file layout, per-entry JSON format, `$schema` pointer, and schema-lint validation
107
+ - [agentme-edr-151](151-ai-eval-standards.md) — AI eval core standards: eval folder structure and Makefile targets (rule `01`); LLM-as-judge binary scoring contract (rule `02`)
108
+ - [agentme-edr-153](153-ai-eval-script.md) — AI eval script: `--type` filtering, entry-first loop, `mock_fixtures`, threshold enforcement, and MLflow conventions
109
+ - [agentme-edr-154](154-ai-eval-report-format.md) — AI eval report format: per-type `report-<type>.md` that consumes this taxonomy's test types
110
+ - [agentme-edr-155](155-ai-eval-repeatability.md) — AI eval repeatability: `REPEAT_COUNT` loop exception, scoring constants (`EVAL_MIN_ACCURACY_REPEATABILITY`, `REPEAT_SEMANTIC_SIMILARITY_SCORE`), scoring methods, `repeatability_accuracy` MLflow metric, report shape, and run cadence
111
+ - [agentme-edr-126](126-pragmatic-hexagonal-architecture.md) — Rule `10`: `_mock` file naming and placement convention for mock adapters referenced by `mock_fixtures`
112
+ - [agentme-edr-501](../governance/501-project-quality-standards.md) — Rule `09` tier-level testing requirements (the only mandated AI testing baseline)
113
+ - [agentme-edr-303](../platform/303-common-targets.md) — Rule `03` `eval-<qualifier>` Makefile convention; rule `03`'s `test-smoke` (distinguished in rule `07`)
114
+ - [agentme-edr-305](../platform/305-environment-variable-configuration.md) — Environment-configuration conventions referenced in rule `09`'s reproducibility disambiguation
115
+ - [agentme-edr-141](141-ai-llm-development-standards.md) — LLM tier definition and mocking utilities referenced by the `mocks allowed` value
116
+ - [agentme-edr-122](122-unit-test-requirements.md) — Unit test requirements underlying the Code-level rows
@@ -1,11 +1,11 @@
1
1
  ---
2
- name: agentme-edr-policy-053-ai-eval-script
3
- description: Defines eval.py script requirements for AI projects — entry-first eval loop, --type test-type filtering, mock_fixtures wiring, human entries, threshold enforcement, and MLflow experiment naming and port assignment. Use when implementing eval scripts. For eval folder structure see agentme-edr-051 rule 01. For the test type taxonomy and mock_fixtures envelope see agentme-edr-052. For mock file naming see agentme-edr-026 rule 10. For report format see agentme-edr-054. For repeatability loop exception see agentme-edr-055.
2
+ name: agentme-edr-policy-153-ai-eval-script
3
+ description: Defines eval.py script requirements for AI projects — entry-first eval loop, --type test-type filtering, mock_fixtures wiring, human entries, threshold enforcement, and MLflow experiment naming and port assignment. Use when implementing eval scripts. For eval folder structure see agentme-edr-151 rule 01. For the test type taxonomy and mock_fixtures envelope see agentme-edr-152. For mock file naming see agentme-edr-126 rule 10. For report format see agentme-edr-154. For repeatability loop exception see agentme-edr-155.
4
4
  apply-to: Python AI projects (LLM, Agent, or Workflow tier) that implement eval testing
5
5
  valid-from: 2026-07-07
6
6
  ---
7
7
 
8
- # agentme-edr-policy-053: AI eval script
8
+ # agentme-edr-policy-153: AI eval script
9
9
 
10
10
  ## Context and Problem Statement
11
11
 
@@ -17,7 +17,7 @@ How should eval scripts load datasets, iterate entries, handle mocking, and prod
17
17
 
18
18
  **Use an entry-first eval loop with `--type` filtering, fresh mock isolation per entry, real LLM providers, and MLflow-backed metrics with explicit per-type thresholds.**
19
19
 
20
- For when evals are required per AI tier, see [agentme-edr-007](../governance/007-project-quality-standards.md) rule `09-ai-project-testing-requirements`.
20
+ For when evals are required per AI tier, see [agentme-edr-501](../governance/501-project-quality-standards.md) rule `09-ai-project-testing-requirements`.
21
21
 
22
22
  ### Details
23
23
 
@@ -25,14 +25,14 @@ For when evals are required per AI tier, see [agentme-edr-007](../governance/007
25
25
 
26
26
  Each `eval.py` script MUST:
27
27
 
28
- - Load the golden dataset from `golden_dataset/` in the same eval folder, following [agentme-edr-050](../data/050-ml-dataset-structure.md) and the entry envelope in [agentme-edr-052](052-ai-test-types-taxonomy.md) rule `02` (one JSON file per entry, `test_types` array, `input`, `expected_output`, optional `mock_fixtures`).
28
+ - Load the golden dataset from `golden_dataset/` in the same eval folder, following [agentme-edr-201](../data/201-ml-dataset-structure.md) and the entry envelope in [agentme-edr-152](152-ai-test-types-taxonomy.md) rule `02` (one JSON file per entry, `test_types` array, `input`, `expected_output`, optional `mock_fixtures`).
29
29
  - Accept a required `--type=<test_type>|all` CLI argument and filter entries whose `test_types` array contains the requested value; `--type=all` includes every entry.
30
30
  - Iterate **entry-first**: for each entry in the filtered set, invoke the real component exactly once; then score that single `actual_output` for every `test_types` value the entry carries that falls within the current `--type` scope — MUST NOT invoke the component more than once per entry per run.
31
- - When an entry contains `mock_fixtures` ([agentme-edr-052](052-ai-test-types-taxonomy.md) rule `02`), configure each named mock adapter with its fixture data BEFORE invoking the component for that entry. Each entry MUST use fresh mock instances so fixture state does not bleed across entries. `mock_fixtures` applies to all test types including `human`. `mock_fixtures` MUST NOT configure LLM adapters — the LLM call MUST be real (see [agentme-edr-052](052-ai-test-types-taxonomy.md) rule `03`). How mock adapters are discovered and instantiated is left to the project; see [agentme-edr-026](026-pragmatic-hexagonal-architecture.md) rule `10` for the `_mock` file naming and placement convention.
31
+ - When an entry contains `mock_fixtures` ([agentme-edr-152](152-ai-test-types-taxonomy.md) rule `02`), configure each named mock adapter with its fixture data BEFORE invoking the component for that entry. Each entry MUST use fresh mock instances so fixture state does not bleed across entries. `mock_fixtures` applies to all test types including `human`. `mock_fixtures` MUST NOT configure LLM adapters — the LLM call MUST be real (see [agentme-edr-152](152-ai-test-types-taxonomy.md) rule `03`). How mock adapters are discovered and instantiated is left to the project; see [agentme-edr-126](126-pragmatic-hexagonal-architecture.md) rule `10` for the `_mock` file naming and placement convention.
32
32
  - Run every component invocation against **real LLM providers** (not mocked responses), to capture model drift.
33
33
  - For `human` entries: invoke the component to capture `actual_output`, export each entry's `input`, `expected_output.human_test` instructions, and `actual_output` into a manual-review checklist (`report-human.md`). MUST NOT invoke an automated scorer and MUST NOT enforce a pass/fail threshold for it. Other `test_types` on the same entry (e.g. `functional`) are still scored automatically.
34
- - After all entries are processed, compute aggregate metrics per test type, log them to a local MLflow experiment (see rule `02`), write one `report-<type>.md` per evaluated test type ([agentme-edr-054](054-ai-eval-report-format.md) rule `01`), and exit with a non-zero status when any metric falls below its defined threshold per [agentme-edr-007](../governance/007-project-quality-standards.md) rule `07-statistical-models-must-have-eval-targets`. The `human` type has no threshold and does not trigger a non-zero exit.
35
- - Compare outputs to expected values using project-defined quality thresholds per test type. Thresholds and all other scoring parameters MUST be declared as constants in `eval.py` — they are design decisions about what constitutes acceptable quality for the component under test, not runtime configuration, and MUST NOT be passed via Makefile variables or CLI flags. Use one of two naming conventions, chosen consistently within an `eval.py`: (a) **per-type constants** — `EVAL_MIN_<METRIC>_<TYPE>` for each test type (e.g. `EVAL_MIN_ACCURACY_FUNCTIONAL = 0.85`, `EVAL_MIN_ACCURACY_REPEATABILITY = 0.8`); or (b) **dict constant** — `EVAL_MIN_<METRIC> = {<type>: <value>}` (e.g. `EVAL_MIN_ACCURACY = {"functional": 0.85, "smoke": 0.85}`). Per-type constants are preferred when each test type has a dedicated `eval.py`; the dict form is preferred when a single `eval.py` handles multiple types. In either convention, `EVAL_MIN_ACCURACY` (as a scalar) MAY be declared as a project-wide default and MUST be used as fallback when no per-type override is defined for the current test type. This Policy does not mandate which test types a project must threshold or what value to use (see [agentme-edr-052](052-ai-test-types-taxonomy.md) rule `06`).
34
+ - After all entries are processed, compute aggregate metrics per test type, log them to a local MLflow experiment (see rule `02`), write one `report-<type>.md` per evaluated test type ([agentme-edr-154](154-ai-eval-report-format.md) rule `01`), and exit with a non-zero status when any metric falls below its defined threshold per [agentme-edr-501](../governance/501-project-quality-standards.md) rule `07-statistical-models-must-have-eval-targets`. The `human` type has no threshold and does not trigger a non-zero exit.
35
+ - Compare outputs to expected values using project-defined quality thresholds per test type. Thresholds and all other scoring parameters MUST be declared as constants in `eval.py` — they are design decisions about what constitutes acceptable quality for the component under test, not runtime configuration, and MUST NOT be passed via Makefile variables or CLI flags. Use one of two naming conventions, chosen consistently within an `eval.py`: (a) **per-type constants** — `EVAL_MIN_<METRIC>_<TYPE>` for each test type (e.g. `EVAL_MIN_ACCURACY_FUNCTIONAL = 0.85`, `EVAL_MIN_ACCURACY_REPEATABILITY = 0.8`); or (b) **dict constant** — `EVAL_MIN_<METRIC> = {<type>: <value>}` (e.g. `EVAL_MIN_ACCURACY = {"functional": 0.85, "smoke": 0.85}`). Per-type constants are preferred when each test type has a dedicated `eval.py`; the dict form is preferred when a single `eval.py` handles multiple types. In either convention, `EVAL_MIN_ACCURACY` (as a scalar) MAY be declared as a project-wide default and MUST be used as fallback when no per-type override is defined for the current test type. This Policy does not mandate which test types a project must threshold or what value to use (see [agentme-edr-152](152-ai-test-types-taxonomy.md) rule `06`).
36
36
 
37
37
  **Example:**
38
38
 
@@ -69,7 +69,7 @@ with mlflow.start_run():
69
69
  # Entry-first loop: invoke each entry exactly once
70
70
  for idx, entry in enumerate(entries, start=1):
71
71
  # Configure mock adapters from mock_fixtures before invocation
72
- # (implementation left to the project — see agentme-edr-026 rule 10)
72
+ # (implementation left to the project — see agentme-edr-126 rule 10)
73
73
  if entry.get("mock_fixtures"):
74
74
  configure_mocks(entry["mock_fixtures"]) # project-defined helper
75
75
 
@@ -120,17 +120,17 @@ with mlflow.start_run():
120
120
 
121
121
  Each `evals/<component>/eval-<name>/Makefile` MUST start its MLflow tracking server on a **unique port** to prevent conflicts when multiple eval Makefiles are run concurrently or in parallel (e.g., in CI or across multiple terminal sessions).
122
122
 
123
- Ports MUST be statically assigned per eval scenario (not per test type) and MUST NOT reuse the default `5000` port (reserved for `dev-mlflow` per [agentme-edr-008](../platform/008-common-targets.md) rule `09-ai-project-dev-targets`). Assign ports starting at `5100` and incrementing by 1 for each additional eval scenario across the entire project.
123
+ Ports MUST be statically assigned per eval scenario (not per test type) and MUST NOT reuse the default `5000` port (reserved for `dev-mlflow` per [agentme-edr-303](../platform/303-common-targets.md) rule `09-ai-project-dev-targets`). Assign ports starting at `5100` and incrementing by 1 for each additional eval scenario across the entire project.
124
124
 
125
125
  The MLflow **experiment** is scoped to the eval scenario: `<component>/<eval-name>` (e.g. `document-review/eval-basic`). Each `mlflow.start_run()` call MUST set a `test_types` tag listing the test types evaluated in that invocation (comma-separated, e.g. `"functional,smoke"` for `--type=all`, `"smoke"` for `--type=smoke`). A remote MLflow server MUST NOT be required — all tracking is local.
126
126
 
127
127
  ## References
128
128
 
129
- - [agentme-edr-051](051-ai-eval-core-standards.md) — AI eval core standards: eval folder structure (rule `01`) and LLM-as-judge binary scoring contract (rule `02`)
130
- - [agentme-edr-054](054-ai-eval-report-format.md) — AI eval report format: `report-<type>.md` template, Wilson CI, and convergence analysis
131
- - [agentme-edr-055](055-ai-eval-repeatability.md) — AI eval repeatability: loop exception to rule `01`'s entry-first constraint, scoring methods, and cadence
132
- - [agentme-edr-052](052-ai-test-types-taxonomy.md) — AI test types taxonomy: `test_types` enum, golden dataset entry envelope (including `mock_fixtures`), and mocking constraints per type
133
- - [agentme-edr-026](026-pragmatic-hexagonal-architecture.md) — Rule `10`: `_mock` file naming and placement convention for mock adapters used in `mock_fixtures`
134
- - [agentme-edr-050](../data/050-ml-dataset-structure.md) — ML dataset structure, per-entry JSON format, and schema-lint validation for golden datasets
135
- - [agentme-edr-007](../governance/007-project-quality-standards.md) — Project quality standards: when evals are required per AI tier (rule `09`) and statistical model eval targets (rule `07`)
136
- - [agentme-edr-008](../platform/008-common-targets.md) — `eval-<qualifier>` Makefile convention (rule `03`) and reserved MLflow port `5000` (rule `09`)
129
+ - [agentme-edr-151](151-ai-eval-standards.md) — AI eval core standards: eval folder structure (rule `01`) and LLM-as-judge binary scoring contract (rule `02`)
130
+ - [agentme-edr-154](154-ai-eval-report-format.md) — AI eval report format: `report-<type>.md` template, Wilson CI, and convergence analysis
131
+ - [agentme-edr-155](155-ai-eval-repeatability.md) — AI eval repeatability: loop exception to rule `01`'s entry-first constraint, scoring methods, and cadence
132
+ - [agentme-edr-152](152-ai-test-types-taxonomy.md) — AI test types taxonomy: `test_types` enum, golden dataset entry envelope (including `mock_fixtures`), and mocking constraints per type
133
+ - [agentme-edr-126](126-pragmatic-hexagonal-architecture.md) — Rule `10`: `_mock` file naming and placement convention for mock adapters used in `mock_fixtures`
134
+ - [agentme-edr-201](../data/201-ml-dataset-structure.md) — ML dataset structure, per-entry JSON format, and schema-lint validation for golden datasets
135
+ - [agentme-edr-501](../governance/501-project-quality-standards.md) — Project quality standards: when evals are required per AI tier (rule `09`) and statistical model eval targets (rule `07`)
136
+ - [agentme-edr-303](../platform/303-common-targets.md) — `eval-<qualifier>` Makefile convention (rule `03`) and reserved MLflow port `5000` (rule `09`)
@@ -1,11 +1,11 @@
1
1
  ---
2
- name: agentme-edr-policy-054-ai-eval-report-format
3
- description: Defines the eval report format for AI projects — report-<type>.md template, Wilson score confidence interval, convergence analysis, and the human-type checklist artifact. Use when generating eval reports from eval scripts. For the eval script that produces these reports see agentme-edr-053. For repeatability report shape see agentme-edr-055 rule 02.
2
+ name: agentme-edr-policy-154-ai-eval-report-format
3
+ description: Defines the eval report format for AI projects — report-<type>.md template, Wilson score confidence interval, convergence analysis, and the human-type checklist artifact. Use when generating eval reports from eval scripts. For the eval script that produces these reports see agentme-edr-153. For repeatability report shape see agentme-edr-155 rule 02.
4
4
  apply-to: Python AI projects (LLM, Agent, or Workflow tier) that implement eval testing
5
5
  valid-from: 2026-07-07
6
6
  ---
7
7
 
8
- # agentme-edr-policy-054: AI eval report format
8
+ # agentme-edr-policy-154: AI eval report format
9
9
 
10
10
  ## Context and Problem Statement
11
11
 
@@ -166,6 +166,6 @@ xychart-beta
166
166
 
167
167
  ## References
168
168
 
169
- - [agentme-edr-053](053-ai-eval-script.md) — AI eval script: the script that produces these reports (rule `01`)
170
- - [agentme-edr-055](055-ai-eval-repeatability.md) — AI eval repeatability: rule `02` defines the adapted report shape for `report-repeatability.md`
171
- - [agentme-edr-051](051-ai-eval-core-standards.md) — AI eval core standards: folder structure (rule `01`) and LLM-as-judge binary scoring (rule `02`)
169
+ - [agentme-edr-153](153-ai-eval-script.md) — AI eval script: the script that produces these reports (rule `01`)
170
+ - [agentme-edr-155](155-ai-eval-repeatability.md) — AI eval repeatability: rule `02` defines the adapted report shape for `report-repeatability.md`
171
+ - [agentme-edr-151](151-ai-eval-standards.md) — AI eval core standards: folder structure (rule `01`) and LLM-as-judge binary scoring (rule `02`)
@@ -1,11 +1,11 @@
1
1
  ---
2
- name: agentme-edr-policy-055-ai-eval-repeatability
3
- description: Defines the repeatability test type for AI evals — REPEAT_COUNT loop exception to the entry-first constraint, semantic-similarity and LLM-as-judge scoring methods, repeatability_accuracy metric, and the repeatability report format and run cadence. Use when implementing repeatability evals. For the entry-first constraint see agentme-edr-053 rule 01. For LLM-as-judge binary output see agentme-edr-051 rule 02. For the base report template see agentme-edr-054 rule 01. For the repeatability test type definition see agentme-edr-052.
2
+ name: agentme-edr-policy-155-ai-eval-repeatability
3
+ description: Defines the repeatability test type for AI evals — REPEAT_COUNT loop exception to the entry-first constraint, semantic-similarity and LLM-as-judge scoring methods, repeatability_accuracy metric, and the repeatability report format and run cadence. Use when implementing repeatability evals. For the entry-first constraint see agentme-edr-153 rule 01. For LLM-as-judge binary output see agentme-edr-151 rule 02. For the base report template see agentme-edr-154 rule 01. For the repeatability test type definition see agentme-edr-152.
4
4
  apply-to: Python AI projects (LLM, Agent, or Workflow tier) that implement repeatability eval testing
5
5
  valid-from: 2026-07-07
6
6
  ---
7
7
 
8
- # agentme-edr-policy-055: AI eval repeatability
8
+ # agentme-edr-policy-155: AI eval repeatability
9
9
 
10
10
  ## Context and Problem Statement
11
11
 
@@ -17,13 +17,13 @@ How should repeatability be measured in evals, and how should its results be rep
17
17
 
18
18
  **Exempt `repeatability` entries from the entry-first constraint, invoking each `REPEAT_COUNT` times and scoring via semantic-similarity or LLM-as-judge; schedule at release cadence rather than per-commit.**
19
19
 
20
- For the `repeatability` test type definition and its disambiguation from `reproducibility`, see [agentme-edr-052](052-ai-test-types-taxonomy.md) rule `09`.
20
+ For the `repeatability` test type definition and its disambiguation from `reproducibility`, see [agentme-edr-152](152-ai-test-types-taxonomy.md) rule `09`.
21
21
 
22
22
  ### Details
23
23
 
24
24
  #### 01-repeatability-eval-loop-exception
25
25
 
26
- Entries whose `test_types` includes `repeatability` are exempt from [agentme-edr-053](053-ai-eval-script.md) rule `01`'s "invoke exactly once per entry" constraint. The following constants MUST be declared in `eval.py` and MUST NOT be exposed as Makefile variables, CLI flags, or stored as per-entry dataset fields:
26
+ Entries whose `test_types` includes `repeatability` are exempt from [agentme-edr-153](153-ai-eval-script.md) rule `01`'s "invoke exactly once per entry" constraint. The following constants MUST be declared in `eval.py` and MUST NOT be exposed as Makefile variables, CLI flags, or stored as per-entry dataset fields:
27
27
 
28
28
  - `REPEAT_COUNT` — number of times each repeatability entry is invoked. SHOULD default to 3-5 for routine CI runs and 10-20 for focused passes on decision-critical or previously-flagged components. Projects SHOULD calibrate the value once per component by plotting cumulative pass rate against repeat count for a few representative entries and picking the point where it plateaus, rather than guessing.
29
29
  - `EVAL_MIN_ACCURACY_REPEATABILITY` — minimum fraction of repeatability entries that must PASS for the eval to exit 0.
@@ -34,7 +34,7 @@ Entries whose `test_types` includes `repeatability` are exempt from [agentme-edr
34
34
  **Choosing the scoring method:** Two approaches are supported, declared as a constant in `eval.py`:
35
35
 
36
36
  - **Semantic-similarity:** Embed all `REPEAT_COUNT` outputs for an entry into vectors and compute the average pairwise cosine similarity. The entry passes (score = 1) if the average similarity ≥ `REPEAT_SEMANTIC_SIMILARITY_SCORE`; otherwise it fails (score = 0). Use for classification or short structured outputs.
37
- - **LLM-as-judge:** Provide all `REPEAT_COUNT` outputs for an entry to an LLM judge (kept at low/zero temperature) that returns 0 (fail) or 1 (pass) directly, based on whether the outputs are sufficiently consistent. No `REPEAT_SEMANTIC_SIMILARITY_SCORE` constant is needed. Use for free-text or complex structured outputs where vector distance is an unreliable proxy for agreement. The judge MUST follow [agentme-edr-051](051-ai-eval-core-standards.md) rule `02`'s binary output contract.
37
+ - **LLM-as-judge:** Provide all `REPEAT_COUNT` outputs for an entry to an LLM judge (kept at low/zero temperature) that returns 0 (fail) or 1 (pass) directly, based on whether the outputs are sufficiently consistent. No `REPEAT_SEMANTIC_SIMILARITY_SCORE` constant is needed. Use for free-text or complex structured outputs where vector distance is an unreliable proxy for agreement. The judge MUST follow [agentme-edr-151](151-ai-eval-standards.md) rule `02`'s binary output contract.
38
38
 
39
39
  **`repeatability_accuracy`:** the fraction of repeatability entries that received PASS (score = 1), logged to MLflow as `repeatability_accuracy`. The eval exits non-zero if `repeatability_accuracy` < `EVAL_MIN_ACCURACY_REPEATABILITY`. Both `repeatability_accuracy` and `repeat_count` MUST be logged to MLflow and included in `report-repeatability.md` (rule `02`).
40
40
 
@@ -56,20 +56,20 @@ if repeatability_accuracy < EVAL_MIN_ACCURACY_REPEATABILITY:
56
56
  raise SystemExit(f"Eval failed: repeatability_accuracy {repeatability_accuracy:.2f} < {EVAL_MIN_ACCURACY_REPEATABILITY}")
57
57
  ```
58
58
 
59
- `mock_fixtures` configuration per [agentme-edr-053](053-ai-eval-script.md) rule `01` applies to each of the `REPEAT_COUNT` invocations. Any prompt or response caching (provider-side or gateway-side) MUST be bypassed for these invocations — a cache hit would return an identical cached response and falsely report perfect stability instead of measuring the model's actual variance.
59
+ `mock_fixtures` configuration per [agentme-edr-153](153-ai-eval-script.md) rule `01` applies to each of the `REPEAT_COUNT` invocations. Any prompt or response caching (provider-side or gateway-side) MUST be bypassed for these invocations — a cache hit would return an identical cached response and falsely report perfect stability instead of measuring the model's actual variance.
60
60
 
61
61
  **Scoping:** this test type MUST NOT be applied to components whose intended behavior is diverse or creative output (e.g. brainstorming, creative writing) — low agreement there is correct behavior, not a defect.
62
62
 
63
63
  #### 02-repeatability-report-and-cadence
64
64
 
65
- `--type=repeatability` MUST produce `report-repeatability.md` with a shape adapted from [agentme-edr-054](054-ai-eval-report-format.md) rule `01`'s template: the header MUST include a **Repeat count:** line stating the `REPEAT_COUNT` value used for the run, alongside the usual Date/Dataset/Script/Thresholds lines. The body MUST have an aggregate row reporting `repeatability_accuracy` (the fraction of entries that PASS — see rule `01`) with a Wilson score interval computed over the number of `repeatability` entries, plus a per-item table listing each entry's individual pass/fail result and, when using semantic-similarity, its computed average pairwise cosine similarity — instead of the `Expected | Actual | Correct` columns used by other types.
65
+ `--type=repeatability` MUST produce `report-repeatability.md` with a shape adapted from [agentme-edr-154](154-ai-eval-report-format.md) rule `01`'s template: the header MUST include a **Repeat count:** line stating the `REPEAT_COUNT` value used for the run, alongside the usual Date/Dataset/Script/Thresholds lines. The body MUST have an aggregate row reporting `repeatability_accuracy` (the fraction of entries that PASS — see rule `01`) with a Wilson score interval computed over the number of `repeatability` entries, plus a per-item table listing each entry's individual pass/fail result and, when using semantic-similarity, its computed average pairwise cosine similarity — instead of the `Expected | Actual | Correct` columns used by other types.
66
66
 
67
- Because `repeatability` entries multiply real LLM-provider calls by `REPEAT_COUNT`, projects SHOULD schedule `make eval-repeatability` at release cadence rather than on every commit, aligned with the Workflow eval cadence in [agentme-edr-007](../governance/007-project-quality-standards.md) rule `09`, rather than treating it as a mandatory per-commit gate.
67
+ Because `repeatability` entries multiply real LLM-provider calls by `REPEAT_COUNT`, projects SHOULD schedule `make eval-repeatability` at release cadence rather than on every commit, aligned with the Workflow eval cadence in [agentme-edr-501](../governance/501-project-quality-standards.md) rule `09`, rather than treating it as a mandatory per-commit gate.
68
68
 
69
69
  ## References
70
70
 
71
- - [agentme-edr-053](053-ai-eval-script.md) — AI eval script: rule `01` defines the entry-first constraint this policy exempts for repeatability entries
72
- - [agentme-edr-051](051-ai-eval-core-standards.md) — AI eval core standards: rule `02` defines the LLM-as-judge binary output contract used by the LLM-as-judge scoring method in rule `01`
73
- - [agentme-edr-054](054-ai-eval-report-format.md) — AI eval report format: rule `01` defines the base report template that `report-repeatability.md` adapts
74
- - [agentme-edr-052](052-ai-test-types-taxonomy.md) — AI test types taxonomy: `repeatability` test type definition and disambiguation from `reproducibility` (rule `09`)
75
- - [agentme-edr-007](../governance/007-project-quality-standards.md) — Project quality standards: Workflow eval cadence (rule `09`) that repeatability runs align with
71
+ - [agentme-edr-153](153-ai-eval-script.md) — AI eval script: rule `01` defines the entry-first constraint this policy exempts for repeatability entries
72
+ - [agentme-edr-151](151-ai-eval-standards.md) — AI eval core standards: rule `02` defines the LLM-as-judge binary output contract used by the LLM-as-judge scoring method in rule `01`
73
+ - [agentme-edr-154](154-ai-eval-report-format.md) — AI eval report format: rule `01` defines the base report template that `report-repeatability.md` adapts
74
+ - [agentme-edr-152](152-ai-test-types-taxonomy.md) — AI test types taxonomy: `repeatability` test type definition and disambiguation from `reproducibility` (rule `09`)
75
+ - [agentme-edr-501](../governance/501-project-quality-standards.md) — Project quality standards: Workflow eval cadence (rule `09`) that repeatability runs align with
@@ -1,5 +1,5 @@
1
1
  ---
2
- name: 004-select-relevant-xdrs
2
+ name: 010-select-relevant-xdrs
3
3
  description: >
4
4
  Analyzes a client repository, extracts the full agentme XDR set, and excludes the records that do
5
5
  not fit the project's structure and workflow needs. Activate this skill when the user asks to
@@ -60,9 +60,9 @@ extraction.
60
60
  1. Start from the full shipped agentme XDR set as the default installation target.
61
61
  2. Reduce that set only by excluding the XDRs that clearly do not fit the repository. Use
62
62
  path-stable identifiers so the extraction command is auditable, for example:
63
- - `.xdrs/agentme/edrs/application/010-golang-project-tooling.md` for non-Go projects
64
- - `.xdrs/agentme/edrs/platform/005-monorepo-structure.md` for non-monorepos
65
- - `.xdrs/agentme/edrs/operations/011-service-health-check-endpoint.md` for projects without
63
+ - `.xdrs/agentme/edrs/application/102-golang-project-tooling.md` for non-Go projects
64
+ - `.xdrs/agentme/edrs/platform/301-monorepo-structure.md` for non-monorepos
65
+ - `.xdrs/agentme/edrs/operations/401-service-health-check-endpoint.md` for projects without
66
66
  a long-running service surface
67
67
  3. If the repository does not want agentme XDRs, stop and explain why instead of forcing an
68
68
  installation.
@@ -109,8 +109,8 @@ extraction.
109
109
  Input: "Install the right agentme XDR presets for this Node.js library"
110
110
  - Inventory the shipped agentme XDR files
111
111
  - Analyze the repository and detect a JavaScript library with Makefiles
112
- - Exclude `.xdrs/agentme/edrs/application/010-golang-project-tooling.md` and `.xdrs/agentme/edrs/operations/011-service-health-check-endpoint.md`
113
- - Run `npx -y agentme extract --output . --all --exclude .xdrs/agentme/edrs/application/010-golang-project-tooling.md --exclude .xdrs/agentme/edrs/operations/011-service-health-check-endpoint.md`
112
+ - Exclude `.xdrs/agentme/edrs/application/102-golang-project-tooling.md` and `.xdrs/agentme/edrs/operations/401-service-health-check-endpoint.md`
113
+ - Run `npx -y agentme extract --output . --all --exclude .xdrs/agentme/edrs/application/102-golang-project-tooling.md --exclude .xdrs/agentme/edrs/operations/401-service-health-check-endpoint.md`
114
114
 
115
115
  Input: "Set up agentme for this repo"
116
116
  - Inventory the shipped agentme XDR files
@@ -134,8 +134,8 @@ Input: "Set up agentme for this repo"
134
134
  ## References
135
135
 
136
136
  - [agentme README](../../../../../../README.md)
137
- - [agentme-edr-003 - JavaScript project tooling and structure](../../003-javascript-project-tooling.md)
138
- - [agentme-edr-005 - Monorepo structure](../../../platform/005-monorepo-structure.md)
139
- - [agentme-edr-007 - Project quality standards](../../../governance/007-project-quality-standards.md)
140
- - [agentme-edr-008 - Common development script names](../../../platform/008-common-targets.md)
137
+ - [agentme-edr-101 - JavaScript project tooling and structure](../../101-javascript-project-tooling.md)
138
+ - [agentme-edr-301 - Monorepo structure](../../../platform/301-monorepo-structure.md)
139
+ - [agentme-edr-501 - Project quality standards](../../../governance/501-project-quality-standards.md)
140
+ - [agentme-edr-303 - Common development script names](../../../platform/303-common-targets.md)
141
141
  - [_core-adr-003 - Skill standards](../../../../../_core/adrs/principles/003-skill-standards.md)
@@ -1,8 +1,8 @@
1
1
  ---
2
- name: 001-create-javascript-project
2
+ name: 050-create-javascript-project
3
3
  description: >
4
4
  Scaffolds the initial boilerplate structure for a JavaScript/TypeScript project following
5
- the standard tooling and layout defined in agentme-edr-003. Activate this skill when the user
5
+ the standard tooling and layout defined in agentme-edr-101. Activate this skill when the user
6
6
  asks to create, scaffold, or initialize a new JavaScript or TypeScript project, npm
7
7
  package, or similar project structure.
8
8
  metadata:
@@ -15,12 +15,12 @@ compatibility: JavaScript/TypeScript, Node.js 18+
15
15
 
16
16
  Creates a complete JavaScript/TypeScript project from scratch. The layout keeps the
17
17
  package self-contained in its module root (`lib/`), organizes internal code following
18
- [agentme-edr-026](../../026-pragmatic-hexagonal-architecture.md) (`adapters/`, `app/`, `shared/`),
18
+ [agentme-edr-126](../../126-pragmatic-hexagonal-architecture.md) (`adapters/`, `app/`, `shared/`),
19
19
  places runnable consumer examples in the sibling `examples/` folder, redirects persistent caches
20
20
  into `.cache/`, and uses Makefiles as the only entry points. Boilerplate is derived from the
21
21
  [filedist](https://github.com/flaviostutz/filedist) project.
22
22
 
23
- Related EDRs: [agentme-edr-003](../../003-javascript-project-tooling.md), [agentme-edr-016](../../../principles/016-cross-language-module-structure.md), [agentme-edr-026](../../026-pragmatic-hexagonal-architecture.md)
23
+ Related EDRs: [agentme-edr-101](../../101-javascript-project-tooling.md), [agentme-edr-016](../../../principles/016-cross-language-module-structure.md), [agentme-edr-126](../../126-pragmatic-hexagonal-architecture.md)
24
24
 
25
25
  ## Instructions
26
26
 
@@ -433,7 +433,7 @@ Review all created files and confirm:
433
433
  - [ ] `lib/eslint.config.mjs` points `parserOptions.project` to `tsconfig.json`
434
434
  - [ ] `lib/README.md` starts with Quick Start and ends with module development commands
435
435
  - [ ] All `[package-name]` placeholders are replaced with the actual name
436
- - [ ] Structure matches the layout in [agentme-edr-003](../../003-javascript-project-tooling.md)
436
+ - [ ] Structure matches the layout in [agentme-edr-101](../../101-javascript-project-tooling.md)
437
437
 
438
438
  ## Examples
439
439
 
@@ -1,8 +1,8 @@
1
1
  ---
2
- name: 003-create-golang-project
2
+ name: 051-create-golang-project
3
3
  description: >
4
4
  Scaffolds the initial boilerplate structure for a Go (Golang) CLI or library project following
5
- the standard tooling and layout defined in agentme-edr-010. Activate this skill when the user
5
+ the standard tooling and layout defined in agentme-edr-102. Activate this skill when the user
6
6
  asks to create, scaffold, or initialize a new Go project, CLI binary, or Go module.
7
7
  metadata:
8
8
  author: flaviostutz
@@ -12,9 +12,9 @@ compatibility: Go 1.21+
12
12
 
13
13
  ## Overview
14
14
 
15
- Creates a complete Go project from scratch, following the layout from [agentme-edr-010](../../010-golang-project-tooling.md) and [agentme-edr-026](../../026-pragmatic-hexagonal-architecture.md). Business logic lives in `app/<feature>/` packages; CLI wiring lives in `adapters/cli/`; outbound integrations live in `adapters/connectors/`; `main.go` is a thin dispatcher. The module root owns its `Makefile`, `README.md`, `dist/`, and `.cache/` folders.
15
+ Creates a complete Go project from scratch, following the layout from [agentme-edr-102](../../102-golang-project-tooling.md) and [agentme-edr-126](../../126-pragmatic-hexagonal-architecture.md). Business logic lives in `app/<feature>/` packages; CLI wiring lives in `adapters/cli/`; outbound integrations live in `adapters/connectors/`; `main.go` is a thin dispatcher. The module root owns its `Makefile`, `README.md`, `dist/`, and `.cache/` folders.
16
16
 
17
- Related EDRs: [agentme-edr-010](../../010-golang-project-tooling.md), [agentme-edr-016](../../../principles/016-cross-language-module-structure.md), [agentme-edr-026](../../026-pragmatic-hexagonal-architecture.md)
17
+ Related EDRs: [agentme-edr-102](../../102-golang-project-tooling.md), [agentme-edr-016](../../../principles/016-cross-language-module-structure.md), [agentme-edr-126](../../126-pragmatic-hexagonal-architecture.md)
18
18
 
19
19
  ## Instructions
20
20
 
@@ -1,8 +1,8 @@
1
1
  ---
2
- name: 005-create-python-project
2
+ name: 052-create-python-project
3
3
  description: >
4
4
  Scaffolds the initial boilerplate structure for a Python project following the standard tooling
5
- and layout defined in agentme-edr-014. Activate this skill when the user asks to create,
5
+ and layout defined in agentme-edr-103. Activate this skill when the user asks to create,
6
6
  scaffold, or initialize a new Python package, CLI, or similar project structure.
7
7
  metadata:
8
8
  author: flaviostutz
@@ -14,11 +14,11 @@ compatibility: Python 3.12+
14
14
 
15
15
  Creates a complete Python project from scratch using Mise, `uv`, `pyproject.toml`, Ruff,
16
16
  ty, Pytest, and Makefiles. The layout keeps the package self-contained under `lib/`,
17
- organizes internal code following [agentme-edr-026](../../026-pragmatic-hexagonal-architecture.md)
17
+ organizes internal code following [agentme-edr-126](../../126-pragmatic-hexagonal-architecture.md)
18
18
  (`adapters/`, `app/`, `shared/`), uses a shared root `.venv/`, redirects persistent caches into
19
19
  `.cache/`, and places runnable consumer projects under the sibling `examples/` folder.
20
20
 
21
- Related EDRs: [agentme-edr-014](../../014-python-project-tooling.md), [agentme-edr-016](../../../principles/016-cross-language-module-structure.md), [agentme-edr-026](../../026-pragmatic-hexagonal-architecture.md)
21
+ Related EDRs: [agentme-edr-103](../../103-python-project-tooling.md), [agentme-edr-016](../../../principles/016-cross-language-module-structure.md), [agentme-edr-126](../../126-pragmatic-hexagonal-architecture.md)
22
22
 
23
23
  ## Instructions
24
24
 
@@ -282,7 +282,7 @@ make test
282
282
 
283
283
  ### Phase 4: Create the package and tests inside `lib/`
284
284
 
285
- Create this baseline structure following [agentme-edr-026](../../026-pragmatic-hexagonal-architecture.md).
285
+ Create this baseline structure following [agentme-edr-126](../../126-pragmatic-hexagonal-architecture.md).
286
286
 
287
287
  **`lib/src/[package_name]/__init__.py`**
288
288
 
@@ -406,5 +406,5 @@ After creating the files:
406
406
 
407
407
  ## References
408
408
 
409
- - [agentme-edr-014](../../014-python-project-tooling.md)
409
+ - [agentme-edr-103](../../103-python-project-tooling.md)
410
410
  - [_core-adr-003 - Skill standards](../../../../../_core/adrs/principles/003-skill-standards.md)
@@ -0,0 +1,113 @@
1
+ ---
2
+ name: 200-run-skill-tests
3
+ description: >
4
+ Executes the SKILL.test.md test scenarios for a given skill, evaluates each assertion against the skill's
5
+ actual output, and reports a pass/fail result per scenario. Activate when the user asks to test, verify,
6
+ or validate a skill, or before merging a PR that modifies a skill or its SKILL.test.md.
7
+ metadata:
8
+ author: flaviostutz
9
+ version: "1.0"
10
+ ---
11
+
12
+ ## Overview
13
+
14
+ Loads `SKILL.test.md` from a skill directory, runs each scenario by invoking the target skill with the specified trigger, evaluates every assertion against the output, and produces a structured test report.
15
+
16
+ ## Instructions
17
+
18
+ ### Phase 1: Locate and Validate SKILL.test.md
19
+
20
+ 1. Accept the skill path from the user (e.g., `.agents/skills/001-review` or `.xdrs/_local/adrs/principles/skills/005-my-skill`). If no path is given, ask: *"Which skill do you want to test? Provide the path to the skill directory."*
21
+ 2. Resolve the path to the directory containing `SKILL.md`. If the path is a symlink, follow it to the real directory.
22
+ 3. Check that `SKILL.test.md` exists in the same directory as `SKILL.md`. If it is missing, output:
23
+
24
+ ```
25
+ ERROR: SKILL.test.md not found in [resolved path].
26
+ Create it following agentme-edr-policy-017 before running tests.
27
+ ```
28
+
29
+ Then halt — do not proceed.
30
+ 4. Read `SKILL.test.md` in full. Parse the frontmatter (`skill`, `skill-version`) and all `### Scenario N:` sections. Each scenario has three sub-sections: `**Trigger / Input**`, `**Expected Behaviour**`, and `**Assertions**`.
31
+ 5. Read `SKILL.md` in full to understand what the skill does. Verify that the `skill-version` in `SKILL.test.md` matches the `version` field in `SKILL.md` metadata. If they differ, output a WARNING and continue.
32
+
33
+ ### Phase 2: Execute Scenarios
34
+
35
+ For each scenario in order:
36
+
37
+ 1. Record the scenario title and number.
38
+ 2. Present the trigger/input to a fresh agent context with the target skill loaded, exactly as written in `**Trigger / Input**`. Do not add context beyond what the trigger specifies.
39
+ 3. Capture the full output from the skill execution.
40
+ 4. Move to Phase 3 for this scenario before executing the next.
41
+
42
+ ### Phase 3: Evaluate Assertions
43
+
44
+ For each assertion in the current scenario:
45
+
46
+ 1. Read the assertion text.
47
+ 2. Determine whether the captured output satisfies the assertion. Apply the following rules:
48
+ - An assertion is PASS if the output unambiguously satisfies the stated condition.
49
+ - An assertion is FAIL if the output clearly does not satisfy the condition.
50
+ - An assertion is INCONCLUSIVE if the output is ambiguous with respect to the condition; treat INCONCLUSIVE as FAIL and note the reason.
51
+ 3. Record the result (PASS / FAIL) and, for FAIL/INCONCLUSIVE, a one-sentence explanation referencing the specific output evidence.
52
+
53
+ ### Phase 4: Report Results
54
+
55
+ After all scenarios are executed, produce the report using this template exactly:
56
+
57
+ ```
58
+ ## Skill Test Report: [skill-name] v[skill-version]
59
+ Tested: [ISO date]
60
+
61
+ ### Scenario 1: [Title]
62
+ - Assertion 1: [PASS|FAIL] — [one-line reason if FAIL]
63
+ - Assertion 2: [PASS|FAIL] — [one-line reason if FAIL]
64
+ ...
65
+ Overall: [PASS|FAIL]
66
+
67
+ ### Scenario 2: [Title]
68
+ ...
69
+
70
+ ---
71
+ ## Summary
72
+ - Scenarios: [total]
73
+ - Passed: [count]
74
+ - Failed: [count]
75
+ - Outcome: [PASS — all scenarios passed | FAIL — [N] scenario(s) failed]
76
+ ```
77
+
78
+ A scenario passes only when every one of its assertions passes. The overall outcome is PASS only when every scenario passes.
79
+
80
+ ### Constraints
81
+
82
+ - MUST NOT modify the skill or its test file.
83
+ - MUST execute scenarios in the order they appear in `SKILL.test.md`.
84
+ - MUST treat INCONCLUSIVE assertions as FAIL.
85
+ - MUST halt and report ERROR if `SKILL.test.md` is absent (Phase 1).
86
+ - MUST follow the report template exactly; do not add commentary outside the template.
87
+
88
+ ## Examples
89
+
90
+ **Input**: "Test the skill at `.agents/skills/001-review`"
91
+
92
+ - Phase 1 resolves the symlink → `.xdrs/_core/adrs/principles/skills/001-review/`
93
+ - Reads `SKILL.test.md`, parses 3 scenarios
94
+ - Executes each scenario with the target skill loaded
95
+ - Reports per-assertion PASS/FAIL and an overall outcome
96
+
97
+ **Input**: "Verify `.xdrs/agentme/edrs/application/skills/050-setup-project` before merging"
98
+
99
+ - Phase 1 reads the real directory (no symlink)
100
+ - If `SKILL.test.md` is missing → ERROR and halt
101
+ - Otherwise proceeds through all phases
102
+
103
+ ## Edge Cases
104
+
105
+ - If the skill path does not exist, output `ERROR: skill directory not found at [path].` and halt.
106
+ - If `SKILL.test.md` contains no scenarios, output `ERROR: SKILL.test.md has no scenarios.` and halt.
107
+ - If the target skill fails to activate (e.g., not registered in VS Code), note this in the report as FAIL with reason "skill could not be activated" and continue to remaining scenarios.
108
+ - If `skill-version` in `SKILL.test.md` does not match `SKILL.md`, emit a WARNING at the top of the report but do not halt.
109
+
110
+ ## References
111
+
112
+ - [`agentme-edr-policy-017`](../../../principles/017-skill-testing.md) — Skill testing mandate and SKILL.test.md format specification
113
+ - [`_core-adr-policy-003`](../../../../../_core/adrs/principles/003-skill-standards.md) — Skill package standards and folder layout