@mutagent/evaluator 0.1.0-alpha.5 → 0.2.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/.claude/skills/mutagent-evaluator/SKILL.md +28 -30
  2. package/.claude/skills/mutagent-evaluator/assets/agents/dataset-builder.md +6 -5
  3. package/.claude/skills/mutagent-evaluator/assets/agents/evaluator.md +93 -30
  4. package/.claude/skills/mutagent-evaluator/assets/code-quality-criteria.yaml +75 -0
  5. package/.claude/skills/mutagent-evaluator/assets/templates/discover-report.template.html +397 -0
  6. package/.claude/skills/mutagent-evaluator/assets/templates/eval-report.template.html +594 -0
  7. package/.claude/skills/mutagent-evaluator/assets/templates/review-report.template.html +560 -0
  8. package/.claude/skills/mutagent-evaluator/golden/judge-trajectory.prose.md +69 -0
  9. package/.claude/skills/mutagent-evaluator/lenses/methodology-critic-lens.md +1 -1
  10. package/.claude/skills/mutagent-evaluator/references/data-registry.md +43 -0
  11. package/.claude/skills/mutagent-evaluator/references/edd-loop.md +6 -6
  12. package/.claude/skills/mutagent-evaluator/references/eval-layers.md +52 -0
  13. package/.claude/skills/mutagent-evaluator/references/eval-stage.md +37 -4
  14. package/.claude/skills/mutagent-evaluator/references/memory-format.md +1 -1
  15. package/.claude/skills/mutagent-evaluator/schemas/methodology-review.schema.yaml +1 -1
  16. package/.claude/skills/mutagent-evaluator/scripts/agent-dispatch.ts +0 -0
  17. package/.claude/skills/mutagent-evaluator/scripts/aggregate-discover.ts +254 -9
  18. package/.claude/skills/mutagent-evaluator/scripts/audience.ts +84 -0
  19. package/.claude/skills/mutagent-evaluator/scripts/build-review-ui.ts +320 -6
  20. package/.claude/skills/mutagent-evaluator/scripts/cli/prep.ts +32 -2
  21. package/.claude/skills/mutagent-evaluator/scripts/cli/profile-subject.ts +1 -1
  22. package/.claude/skills/mutagent-evaluator/scripts/code-eval-library.ts +201 -0
  23. package/.claude/skills/mutagent-evaluator/scripts/code-quality-verdict.ts +60 -68
  24. package/.claude/skills/mutagent-evaluator/scripts/config/schema.ts +3 -3
  25. package/.claude/skills/mutagent-evaluator/scripts/contracts/agentspec-evals.ts +101 -39
  26. package/.claude/skills/mutagent-evaluator/scripts/contracts/eval-matrix.ts +326 -3
  27. package/.claude/skills/mutagent-evaluator/scripts/contracts/eval-types.ts +30 -4
  28. package/.claude/skills/mutagent-evaluator/scripts/determine-outcome.ts +8 -0
  29. package/.claude/skills/mutagent-evaluator/scripts/edd/change-request.ts +1 -1
  30. package/.claude/skills/mutagent-evaluator/scripts/edd/edd-types.ts +2 -2
  31. package/.claude/skills/mutagent-evaluator/scripts/judge-prompt-template.ts +144 -15
  32. package/.claude/skills/mutagent-evaluator/scripts/materialize-dataset.ts +130 -68
  33. package/.claude/skills/mutagent-evaluator/scripts/matrix-judge.ts +219 -1
  34. package/.claude/skills/mutagent-evaluator/scripts/memory/append.ts +1 -1
  35. package/.claude/skills/mutagent-evaluator/scripts/memory/ratify.ts +165 -0
  36. package/.claude/skills/mutagent-evaluator/scripts/merge-criteria.ts +849 -0
  37. package/.claude/skills/mutagent-evaluator/scripts/prep-tasks.ts +15 -4
  38. package/.claude/skills/mutagent-evaluator/scripts/read-manifest.ts +120 -0
  39. package/.claude/skills/mutagent-evaluator/scripts/read-unitf-traces.ts +34 -2
  40. package/.claude/skills/mutagent-evaluator/scripts/render-build-cards.ts +37 -0
  41. package/.claude/skills/mutagent-evaluator/scripts/render-discover-report-v3.ts +1155 -0
  42. package/.claude/skills/mutagent-evaluator/scripts/render-eval-report-v3.ts +610 -0
  43. package/.claude/skills/mutagent-evaluator/scripts/render-eval-report.ts +11 -92
  44. package/.claude/skills/mutagent-evaluator/scripts/render-review-report-v3.ts +691 -0
  45. package/.claude/skills/mutagent-evaluator/scripts/report-fragments.ts +173 -0
  46. package/.claude/skills/mutagent-evaluator/scripts/route-failures.ts +12 -9
  47. package/.claude/skills/mutagent-evaluator/scripts/run-evaluate.ts +46 -1
  48. package/.claude/skills/mutagent-evaluator/scripts/run-pipeline.ts +39 -4
  49. package/.claude/skills/mutagent-evaluator/scripts/run-review.ts +266 -0
  50. package/.claude/skills/mutagent-evaluator/scripts/subject-profile.ts +82 -0
  51. package/.claude/skills/mutagent-evaluator/scripts/sync-eval-criteria.ts +2 -2
  52. package/.claude/skills/mutagent-evaluator/scripts/unitf-to-evaltrace.ts +64 -21
  53. package/.claude/skills/mutagent-evaluator/scripts/verify-render.ts +728 -0
  54. package/bin/mutagent-cli.mjs +9 -5
  55. package/package.json +2 -2
@@ -0,0 +1,69 @@
1
+ # golden/judge-trajectory.prose.md — the C-PIN prose surface (VERSIONED anchor)
2
+
3
+ > The #mode-judge-trajectory judge constructs its prompt AT REASON-TIME from the
4
+ > packet + this mode's prose — so THIS prose is the load-bearing C-PIN surface.
5
+ > Any change to the workflow below is a judge-behavior change and MUST bump the
6
+ > version here (the W-gate byte-identity proof anchors against this snapshot).
7
+
8
+ ## v3.1 — 2026-07-24 · REF-CONTRACT HARDENING (the first-real-run verify-surface flip)
9
+
10
+ Change vs v3.0 (ONE workflow line — GATHER): the ref contract is made explicit
11
+ after the first real run showed judge-cited packet-shaped refs reading as DEAD
12
+ at deterministic re-verify (both verifier-UPHELD HIGH fails silently downgraded
13
+ → gate flipped fail→incomplete). Now: `ref.obs` = the packet's `trajectoryId`
14
+ (NEVER an observation name — batch-order first-match can locate the wrong
15
+ trajectory), `ref.path` = packet-shaped `trajectory.N`/`transcript.N` OR
16
+ trace-shaped `observations.N` (both re-resolve — the verifier reconstructs the
17
+ packet views via `verifySituationViews`), dot form canonical, brackets
18
+ tolerated (`readPath` normalizes). VERBATIM snapshot of the mode workflow
19
+ follows.
20
+
21
+ ```yaml
22
+ - evaluate:
23
+ meta:
24
+ what: "Score ONE trajectory against the WHOLE eval matrix (was the eval-matrix-judge agent). The DEFAULT/headline judging cell."
25
+ does: "Reads a MatrixPacket (the whole matrix + one trajectory + transcript + pinned envelope), BUILDS its judging prompt from the packet + the shared write-judge-prompt.md, scores every criterion for that trajectory, writes a per-trajectory MatrixVerdictFile."
26
+ why: "Per-TRAJECTORY fan-out (one judge scores the whole matrix for one session) = high throughput across many sessions. EV-048. This is the headline *evaluate path."
27
+ how: "Unlike the criterion axis, NO script renders the prompt — the parent's matrix-judge.ts PREPs only a DATA packet (keyed by trajectoryKey); THIS mode constructs the prompt at reason-time. Its prompt-construction prose is therefore the load-bearing C-PIN surface (golden/judge-trajectory.prose.md)."
28
+ display: "Score one trajectory against the WHOLE eval matrix (headline)"
29
+ description: |
30
+ PURPOSE: the DEFAULT headline judging cell — one judge per trajectory scores
31
+ the entire eval matrix for that session → per-criterion verdicts. Critique-
32
+ before-verdict, binary, inaction-can-be-success, whole-matrix-per-trajectory.
33
+
34
+ USAGE: dispatched by the parent, MASS-PARALLEL (one per trajectory). Reads
35
+ <trajectory_key>.packet.json (a MatrixPacket — DATA, NOT a rendered prompt);
36
+ BUILDS the prompt from it + write-judge-prompt.md; writes the MatrixVerdictFile.
37
+ dispatch: { mode: judge, axis: trajectory }
38
+ pre_gate.loads:
39
+ - "references/write-judge-prompt.md" # SHARED 4-component contract (both judge axes; now BIND-before-judge)
40
+ - "references/grounded-adjudication.md" # GA doctrine: bind · gather refs · typed assumptions · abstain · verify
41
+ - "references/workflows/orchestrator-protocol.md" # Step 2 dispatch FSM
42
+ workflow:
43
+ - "Pre-read references/write-judge-prompt.md (the 4-component judging contract — your lens) + references/grounded-adjudication.md"
44
+ - "Read your assigned <trajectory_key>.packet.json (MatrixPacket: subject · trajectoryId · WHOLE matrix · trajectory · transcript · pinned envelope) — judge exactly this, never re-derive the data"
45
+ - "Frame the trajectory in its ROUTE/intended-outcome (CONTEXT) before scoring"
46
+ - "v3 LAYERED WALK (D1=B+, operator-frozen): the walk IS a fixed layer sequence you run INSIDE this one dispatch. Honor `packet.layerScope` when present (absent/empty = FULL walk). The phases: (L0 code checks) if the code-eval library is present at scripts/code-eval-library.ts, RUN it yourself via Bash over the packet's trajectory JSON and record each hit as codeEvalHits[{pattern,anchor,detail}] — a fired pattern is evidence, not a verdict; (L1 outcome) classify the SCENARIO from the inbound intent + tool inventory, derive its expectedExit[] terminal actions, glance ONLY at end-state evidence and emit layerVerdicts[{layer:'L1',verdict,scenario,expectedExit,refs,note}]; (STOPPING RULES) if L1 passes AND every criterion is answerable from evidence gathered so far → you may set earlyExit:'L1' and mark L2-L4 as {verdict:'skipped'} — but a criterion may NEVER go unverdicted because the agent did the right thing; if ANY criterion is unanswerable OR L1 fails/undecidable → descend; (L2 trajectory) actual-vs-expected decision path, read from the END, emit layerVerdicts L2 (+ divergence in localize); (L3 tool outputs) per-observation structural validity + completeness — external-data correctness is UNVERIFIABLE (fence), a failed tool makes everything downstream suspect; emit layerVerdicts L3 (+firstProblemOrigin); (L4 context) for each critical operational send: was required context retrieved AND present in history AT THAT TIME (chronology-sorted!) — distinguish gapKind retrieval (never fetched) vs attention (present, ignored); emit layerVerdicts L4. Record layersEngaged. CONFLICT DISCIPLINE (OT-1): when your own layers disagree (e.g. L1 pass vs L2 fail) emit BOTH verdicts honestly with a note — NEVER reconcile them yourself; the aggregate surfaces the conflict for a human."
47
+ - "Score EVERY criterion in the matrix for THIS trajectory: read only what the row needs (judgeInputs); compare against statement + passCondition — criteria are the SUPER-LAYER: they may bind evidence from ANY walk phase"
48
+ - "BIND (L1) per row: each criterion TERM must resolve in THIS trajectory; an unbound term ⇒ uncertain + blockedBy:{kind:factual-intent} (INDETERMINATE), ABSTAIN — never fail"
49
+ - "GATHER: cite a structured ref {obs,path,value} for the claim AND any absence (ref.obs = the packet's trajectoryId, NEVER an observation name; ref.path = packet-shaped trajectory.N/transcript.N OR trace-shaped observations.N — dot form canonical, brackets tolerated); surface TYPED assumptions for any ungroundable premise → uncertain(blockedBy)"
50
+ - "Critique BEFORE verdict; commit to result ∈ {pass,fail} (uncertain when the INPUTS can't decide — abstain-on-silence); binary, severity is the row's"
51
+ - "Inaction-can-be-success: a correct HOLD (no send during a non-critical outbound_guard) is a PASS even with zero tool calls"
52
+ - "Write <trajectory_key>.verdict.json (a MatrixVerdictFile {trajectoryId, judgeModel, temperature:0, verdicts[]} — each verdict may carry refs?/assumptions?/blockedBy?)"
53
+ - "EMIT-CONTRACT (HARD, WS-1): on every COMPLETE-fidelity trajectory you MUST also persist the §9.4 walk — `understanding` (M2), `expectedTrajectory` (M3), `agentSteps` (the target step lane), and `judgeSteps` (your anchored reasoning). These are NOT optional: the parent runs a machine `assessEmitCompleteness` gate (scripts/emit-completeness.ts) that LOUDLY flags any wholly-dropped field, and a verdict missing them STARVES the report's Trajectory (§2) + Self-Eval (§5) tabs. `agentSteps` is factual trace data (reconstruct it from the ordered tool steps you were given — order by startTime, the observations array is reverse-chronological); M2/M3/judgeSteps are YOUR reasoning. Only an INCOMPLETE (node-1) trajectory is exempt (it emits verdicts:[] and no walk)."
54
+ - "v3 EMIT-CONTRACT ADDITIONS (E-NEW): alongside the §9.4 walk, persist layerVerdicts[] (one per engaged layer, skipped ones marked), earlyExit?, layersEngaged, codeEvalHits[] (when the library ran), and the emissions SELF-MANIFEST {emitted[], missing[{key,reason}]} — list honestly what you gathered vs what this contract expects; a missing emission with a stated reason is a NAMED degrade, a silent drop is a defect (MR-5 catches it same-run)."
55
+ - "Verdicts are independently VERIFIED (#mode-verify · ≠ judge · downgrade-only); a CRIT/HIGH uncertain rolls the run up to INCOMPLETE at the gate (no false-green)"
56
+ compresses:
57
+ - "read MatrixPacket DATA; frame in ROUTE (CONTEXT)"
58
+ - "build judging prompt from packet + write-judge-prompt.md"
59
+ - "BIND terms · GATHER refs + typed assumptions · abstain on silence"
60
+ - "score whole matrix per trajectory, critique-before-verdict"
61
+ - "emit MatrixVerdictFile (then independent VERIFY, downgrade-only)"
62
+ preserves: "the eval-matrix-judge discipline (the headline cell; prompt-construction prose at golden/judge-trajectory.prose.md) — folded INLINE as 'Mode: judge — axis trajectory' (below); the standalone assets/agents/eval-matrix-judge.md was RETIRED in the 5→3 consolidation (Phase 3a, df6a6e8c8). This file is its canonical home."
63
+ ```
64
+
65
+ ## history
66
+
67
+ - v3.1 (2026-07-24): GATHER ref-contract hardening — obs=trajectoryId, packet-OR-trace-shaped paths, dot canonical/brackets tolerated (fixes the verify-surface fail→incomplete flip found on the first real-trace run).
68
+ - v3.0 (2026-07-24): layered walk + v3 emit-contract (evaluator-v3, operator-frozen).
69
+ - v2.2 and earlier: prose lived inline in evaluator.md only; no on-disk snapshot existed (this file materializes the long-referenced golden anchor).
@@ -44,7 +44,7 @@ findings:
44
44
  - mr: MR-7
45
45
  verdict: <fit | mis-fit | inefficient>
46
46
  observation: <what the run actually did at this methodology decision>
47
- recommendation: <concrete rearrange/improve proposal>
47
+ recommendation: <concrete rearrange/optimize proposal>
48
48
  impactRank: <1=highest> # ranked self-feedback (MR-5)
49
49
  advisory: true # NEVER gates the run
50
50
  ```
@@ -0,0 +1,43 @@
1
+ # The data registry — every entity this skill can produce or consume
2
+
3
+ > The agent's general knowledge of available entities (the resilience mechanism from
4
+ > the data-availability audit §⓪). Purpose: so that when an expected entity is ABSENT,
5
+ > the absence is **NAMED in words** — never a silent blank, never a fabricated value.
6
+ > A renderer that cannot source a field says so ("not recorded by this run"); a judge
7
+ > that cannot gather an emission lists it in its `emissions.missing[]` self-manifest.
8
+
9
+ ## Run artifacts (the `.mutagent/evaluator/runs/<runId>/` dot-root, gitignored)
10
+
11
+ | entity | produced by | consumed by | absence → |
12
+ |---|---|---|---|
13
+ | `run-input.full.json` | run PREP (`run-evaluate`) | AGGREGATE, the report renderers | a run cannot aggregate — fail loud |
14
+ | `packets/<trajectoryKey>.packet.json` | `prepMatrixPackets` | one dispatched judge each | the readiness gate throws (no silent partial roll-up) |
15
+ | `verdicts/<trajectoryKey>.verdict.json` | a dispatched `#mode-judge-trajectory` agent | AGGREGATE | same — every judged trajectory MUST have one |
16
+ | `verdicts/*.verify.json` | a dispatched `#mode-verify` reviewer | the T3 independent-verify ledger | self-verify fallback (a NAMED degrade, not a silent downgrade) |
17
+ | `near-duplicate-decisions.json` | `aggregateDiscover` | the next run's brief (no re-proposal) | first run — key absent, never an empty claim |
18
+ | `.mutagent/evaluator/reports/<runId>/{evaluation,discovery,review}-report.html` | the renderers | the operator | the run wrote no report — fail loud |
19
+
20
+ ## Judge emissions (the verdict-file fields — E-NEW)
21
+
22
+ | field | shape | honesty rule |
23
+ |---|---|---|
24
+ | `layerVerdicts[]` | `{layer, verdict, refs?, note?, scenario?, expectedExit?, gapKind?, firstProblemOrigin?, divergence?}` | skipped ≠ undecidable; L0 may be `fired` (a candidate, never a verdict) |
25
+ | `verdicts[]` | per-criterion `{critique, result, confidence, refs?, assumptions?, blockedBy?}` | a DECIDED verdict carries ≥1 re-resolvable ref; an `uncertain` carries `blockedBy` (na for grounding) |
26
+ | `denseMap` | `{criterionId: pass\|fail\|uncertain\|na}` | EVERY criterion appears; `na` ≠ fail |
27
+ | `naRationale` | `{criterionId: why}` | the reason a cell is `na` — the completeness law's third path carries its rationale |
28
+ | `emissions` | `{emitted[], missing[{key,reason}]}` | a missing emission WITH a stated reason is a NAMED degrade; a silent drop is a defect (MR-5 catches it) |
29
+ | `codeEvalHits[]` | `{pattern, anchor, detail}` | L0 fired patterns — evidence, not a verdict |
30
+ | `localize` | `{root, evidence?, criteria?, independentRoots?, conflict?, routing?, …}` | the root-not-symptom; `conflict` is the judge's OT-1 statement — surfaced, never reconciled |
31
+
32
+ ## Subject profile + criteria
33
+
34
+ | entity | produced by | note |
35
+ |---|---|---|
36
+ | subject profile | `buildSubjectProfile` | identity · purpose · tools · scope — GIVEN (code access) or reconstructed |
37
+ | `criteria[]` (the eval matrix) | the suite / `*discover` | each may carry `layer` (which evidence layer it binds to) — additive, optional, defaults to `"criteria"` |
38
+ | living suite + provenance | `growLivingSuite` | append-only, monotonic; near-duplicates MERGE (no data loss) |
39
+ | pending near-duplicate decisions | `merge-criteria` | durable, order-independent identity; stays open until a ruling |
40
+
41
+ **The rule that binds every row of this registry:** an absence is stated, never inferred
42
+ into a number. `0` and "not recorded" are different facts, and a report that shows the
43
+ first when it means the second is manufacturing evidence.
@@ -1,11 +1,11 @@
1
- # edd-loop — the ADL ③ IMPROVE / Eval-Driven-Development loop (F18 + F19)
1
+ # edd-loop — the ADL ③ OPTIMIZE / Eval-Driven-Development loop (F18 + F19)
2
2
 
3
- > **Loaded by:** the evaluator `improve` mode (`assets/agents/evaluator.md#mode-improve`). Load on demand.
3
+ > **Loaded by:** the evaluator `optimize` mode (`assets/agents/evaluator.md#mode-optimize`). Load on demand.
4
4
  > **Backed by:** `scripts/edd/variance-gate.ts` (F19) · `scripts/edd/change-request.ts` (F18) ·
5
5
  > `scripts/edd/edd-types.ts` + `schemas/edd-change-request.schema.yaml` (the contract).
6
6
  > **Sibling refs:** `grounded-adjudication.md` (every ask is grounded) · `validate-evaluator.md` (trust the judge first).
7
7
 
8
- The IMPROVE stage closes the **PR-011 spec↔impl↔eval triad**. A judge that only reports is half a
8
+ The OPTIMIZE stage closes the **PR-011 spec↔impl↔eval triad**. A judge that only reports is half a
9
9
  loop; EDD drives the subject to **full green** while the evaluator stays **judge-only (EV-051)** — it
10
10
  **REQUESTS** the `ai-engineer` to amend the Agent/AgentSpec and **re-evals**, but never
11
11
  patches the subject itself.
@@ -100,11 +100,11 @@ controller never reads a clock) → the decision is PURE + deterministic (C-PIN)
100
100
  ## Assumed eval-runner interface (the clean seam — REBUILD nothing)
101
101
 
102
102
  The EVAL engine (dataset build · the eval runner · Path A/B) is built **in parallel** (sibling
103
- worktree). The `improve` mode is **additive**: it orchestrates the LOOP around the runner and assumes
103
+ worktree). The `optimize` mode is **additive**: it orchestrates the LOOP around the runner and assumes
104
104
  this interface — it does not re-implement it.
105
105
 
106
106
  ```
107
- // the runner the improve mode CONSUMES (owned by the eval engine, e.g. run-evaluate.ts):
107
+ // the runner the optimize mode CONSUMES (owned by the eval engine, e.g. run-evaluate.ts):
108
108
  runOnce(caseIds: string[]) -> Array<{
109
109
  caseId: string;
110
110
  criterionId: string;
@@ -116,7 +116,7 @@ runOnce(caseIds: string[]) -> Array<{
116
116
  - **repeat-N** = `runOnce` called N times over the SAME `caseIds`; the per-rerun verdicts feed
117
117
  `CaseVarianceObservation.verdicts[]` (+ `trajectories[]`) → `evaluateVarianceGate`.
118
118
  - **accuracy** = `runOnce` over the FULL dataset → per-case verdicts → an accuracy ratio vs target.
119
- - the runner is **C-PIN** (pinned model + temperature 0). The improve mode adds **no** provider call.
119
+ - the runner is **C-PIN** (pinned model + temperature 0). The optimize mode adds **no** provider call.
120
120
 
121
121
  If the sibling runner exposes a different shape, adapt at the seam (map its result into
122
122
  `CaseVarianceObservation` / the accuracy ratio) — keep the gate + loop modules untouched. The gate
@@ -0,0 +1,52 @@
1
+ # The evidence layers (L0–L4) — how the layered judge reads a trajectory
2
+
3
+ > The know-how reference for the layered walk. Load when you need to reason about
4
+ > **what a layer can and cannot decide**, why the judge descends, and what the
5
+ > stopping rules permit. The judge emits one `LayerVerdict` per engaged layer; this
6
+ > file is the map between a layer id and its meaning. Live consumers: the judge's
7
+ > `layerVerdicts`, the aggregate's `foldLayerVerdicts` / `detectLayerConflicts`, and
8
+ > the report's layer matrix.
9
+
10
+ ## The ladder
11
+
12
+ The judge walks **cheapest-first, descending only when evidence requires**. Each layer
13
+ answers a narrower question than the one above; a layer's verdict is *evidence*, never
14
+ the run verdict on its own (the GATE is driven by criteria severity — D2).
15
+
16
+ | layer | name | question it can settle | what it CANNOT decide (the fence) |
17
+ |---|---|---|---|
18
+ | **L0** | code checks | did a deterministic pattern fire? (fault passthrough · error output · malformed structure · unguarded send · out-of-order gate) | a fired pattern is a **candidate**, not a verdict — "code-detectable ≠ incorrect". Emits `codeEvalHits`, verdict `fired` — never pass/fail. |
19
+ | **L1** | outcome | did the agent's terminal action belong to the scenario's expected exit set? (scenario → `expectedExit[]`) | anything about the *path*: a correct outcome can hide an unguarded send, a skipped review, an ignored flag. |
20
+ | **L2** | trajectory | did the actual decision path match the expected one? (actual-vs-expected, read from the END) | whether a *tool's* output was trustworthy — that is L3. A divergence here is the first sign the outcome misleads. |
21
+ | **L3** | tool outputs | were the tool outputs structurally valid + complete? (per-observation) | **external-data correctness is UNVERIFIABLE** — a well-formed output can be factually wrong. A failed tool makes everything downstream suspect. |
22
+ | **L4** | context | was required context retrieved AND present in history at the moment of the decision? (chronology-sorted) | distinguish **`gapKind: retrieval`** (never fetched) from **`attention`** (present, ignored) — the fixes differ. |
23
+
24
+ ## Criteria are the super-layer
25
+
26
+ Criteria do not live *in* one layer — they are the **super-layer** and may bind evidence
27
+ from any phase (e.g. a reply-grounding criterion reads L3 tool outputs AND L1 the reply).
28
+ Two hard rules:
29
+
30
+ - **a criterion may NEVER go unverdicted because the agent did the right thing** — `na`
31
+ (not-applicable) is a real accounting path with a rationale, never a silent skip
32
+ (`assertNoUnverdictedCriterion` THROWS on a missing cell).
33
+ - **`na` is not `uncertain`.** `na` = the criterion's scoping predicate is positively
34
+ falsified for this trajectory (never fail). `uncertain` = the inputs could not decide
35
+ (a CRIT/HIGH uncertain rolls the run INCOMPLETE-wards, never green).
36
+
37
+ ## The stopping rules (when the walk may exit early)
38
+
39
+ - If **L1 passes AND every criterion is answerable** from evidence gathered so far → the
40
+ walk may set `earlyExit: 'L1'` and mark L2–L4 `skipped`. `skipped ≠ undecidable`.
41
+ - If **any** criterion is unanswerable, OR L1 fails / is undecidable → **descend.** A
42
+ walk must not stop while a criterion is unanswerable (that is how a false-green forms).
43
+ - `layersEngaged` records how deep the walk actually went (the economy stat — MR-3).
44
+
45
+ ## Conflict discipline (OT-1)
46
+
47
+ When layers disagree — e.g. **L1 pass but L2 fail** (the outcome was right, the path
48
+ unguarded) — the judge emits **both** verdicts honestly with a note and **never
49
+ reconciles them itself**. The aggregate (`detectLayerConflicts`) surfaces the conflict
50
+ to a human calibration queue; the GATE never reads it (it is driven by criteria
51
+ severity). The judge's own statement of the tension lives in `localize.conflict`; the
52
+ aggregate's cross-check is the layer notes. Both render; neither is auto-resolved.
@@ -21,7 +21,7 @@ spine (`error-analysis.md` · `write-judge-prompt.md` · `validate-evaluator.md`
21
21
  | Input | Shape | Used for |
22
22
  |-------|-------|----------|
23
23
  | the built agent | a target-framework impl (e.g. a Mastra/TS agent) OR a harness agent | the eval SUBJECT |
24
- | `agentspec.definition.evals` | `{ success_criteria[], scenarios[], dataset_categories[] }` | dataset materialization (F8) + criteria |
24
+ | `agentspec.spec.evaluation` | `{ criteria[], scenarios[], datasets[] }` (0.3.0, D18/D19 — datasets carry dataset-local `categories[]` + `caseDimensions` + real `items[]`) | dataset materialization (F8) + criteria |
25
25
  | `agentspec.build` | `{ target_framework, runtime, target_eval_framework }` | the target-conditional engine fork (F9) |
26
26
 
27
27
  The skill is **standalone** — it NEVER imports the agentspec schema. It consumes
@@ -38,8 +38,37 @@ After `*build`, OFFER the two derivations (AskUserQuestion, parent session only)
38
38
  1. **`*build-dataset`** — materialize + grow the golden dataset from the spec.
39
39
  2. **`*build-evals`** — build the eval suite (and pick the engine).
40
40
 
41
+ **State what the stage will produce, up front (EVAL-1).** Before running either build,
42
+ EMIT the "what this stage will produce" notice (`renderStageProducesNotice`, `scripts/
43
+ render-build-cards.ts`) naming the deliverables — a **dataset** (golden cases + the sink
44
+ file) and/or the **eval criteria** (the judges/code-checks + their sink) — so the operator
45
+ knows what's coming. Notification only — **no approval gate** (operator constraint): once a
46
+ build is triggered, run it; don't ask for edit-approval, just present clearly.
47
+
41
48
  Let the user run either/both. They are independent; `*evaluate` consumes both.
42
49
 
50
+ **Either order — no hard sequence (EVAL-4).** There is NO build-time ordering rule between
51
+ the dataset and the criteria: you may build the **criteria first** and derive a dataset to
52
+ exercise them (when you can define "good" up front), or build a **dataset first** and mine
53
+ criteria from it (when you bring examples of good/bad and fit criteria to them). Start from
54
+ either end. The ONLY directional constraint is a RUN-TIME prerequisite, not a build order:
55
+ `*evaluate` cannot run against no dataset, so it auto-routes to `*build-dataset` first (below).
56
+ Do not mistake that run-time prerequisite for a build-time sequence.
57
+
58
+ **The three dataset pipelines — kept SEPARATE (EVAL-2).** Do NOT fuse the source paths into
59
+ one step (on a cold start there are no traces to ground against). They stay distinct:
60
+ 1. **cold-from-definition** — materialize the dataset from the agentspec (`materialize-dataset.ts`
61
+ `materializeFromAgentspec`). The only path with no traces.
62
+ 2. **from-labeled-traces** — grow the set from already-labeled traces
63
+ (`derive-dataset.ts` `deriveRegressionCases` / `growRegressionSet`).
64
+ 3. **seed-then-ground** — seed from the definition, THEN ground from traces (definition seeds →
65
+ traces ground). Reuses the shared monotonic merge floor (`build-dataset.ts` — nothing is ever
66
+ dropped), so the paths can be combined later without losing cases.
67
+
68
+ **Precedence — scored traces outrank a stale spec (EVAL-2).** When BOTH a spec and traces
69
+ carrying feedback/scores exist for the same case, the **scored traces WIN**: real observed
70
+ outcomes outrank a possibly-stale definition. The spec seeds; the scored traces correct.
71
+
43
72
  **Dataset gate (dogfood H5).** `*evaluate` REQUIRES a materialized dataset. If `*build-evals` runs
44
73
  (or `*evaluate` is invoked) with no dataset present, do NOT dead-end — auto-route to `*build-dataset`
45
74
  (interactive) first, then resume. Never leave the user with an eval suite and nothing to run it on.
@@ -126,9 +155,13 @@ Stream PROGRESS as wireframe cards for BOTH `*build-dataset` and `*build-evals`
126
155
 
127
156
  - `renderBuildDatasetProgressCard` / `renderBuildEvalsProgressCard` — phase +
128
157
  progress bar + counters, emitted per phase (F13/F16).
129
- - After each, emit the VERBOSE entity card (F22): `renderDatasetEntityCard`
130
- (per-category breakdown · version · sink) · `renderEvalsEntityCard` (engine ·
131
- SURFACED CC/log-sink dependency · criteria).
158
+ - **REQUIRED emission (EVAL-1).** After each build, you MUST emit the VERBOSE entity
159
+ card (F22) this is not optional: `renderDatasetEntityCard` (per-category breakdown ·
160
+ version · sink) · `renderEvalsEntityCard` (engine · SURFACED CC/log-sink dependency ·
161
+ criteria). A build that reports and a build that stays silent must NOT look identical
162
+ (the dogfood symptom). The renderers WARN loudly (`⚠ EMPTY`) when a build produced no
163
+ items/criteria, so a silent no-op build can never render as a success. No approval
164
+ branch — notification only.
132
165
 
133
166
  ---
134
167
 
@@ -21,7 +21,7 @@ name: <short-kebab-case-slug>
21
21
  description: <one-line summary — used to decide relevance during recall>
22
22
  metadata:
23
23
  type: user | feedback | project | reference # Claude Code's four types
24
- lifecycle: spec|build|evaluate|diagnose|improve|general # ADL extension
24
+ lifecycle: spec|build|evaluate|diagnose|optimize|general # ADL extension
25
25
  created: YYYY-MM-DD # REQUIRED · every memory is DATED (stale-pruning)
26
26
  ---
27
27
 
@@ -5,7 +5,7 @@
5
5
  # This is the SECOND requirement family: FITNESS, not conformance. It does not
6
6
  # pass/fail a component; it interrogates the methodology's own decision-tree,
7
7
  # workflow structure, and data-flow, and emits ranked recommendations to
8
- # rearrange / improve the process. MR-1..9 (Tab-4 lens reads these).
8
+ # rearrange / optimize the process. MR-1..9 (Tab-4 lens reads these).
9
9
  # ============================================================================
10
10
  $id: mutagent-evaluator/methodology-review
11
11
  title: Subject Methodology-Review Rubric