@mutagent/evaluator 0.1.0-alpha.6 → 0.2.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/mutagent-evaluator/SKILL.md +23 -25
- package/.claude/skills/mutagent-evaluator/assets/agents/dataset-builder.md +6 -5
- package/.claude/skills/mutagent-evaluator/assets/agents/evaluator.md +77 -14
- package/.claude/skills/mutagent-evaluator/assets/templates/discover-report.template.html +397 -0
- package/.claude/skills/mutagent-evaluator/assets/templates/eval-report.template.html +594 -0
- package/.claude/skills/mutagent-evaluator/assets/templates/review-report.template.html +560 -0
- package/.claude/skills/mutagent-evaluator/golden/judge-trajectory.prose.md +69 -0
- package/.claude/skills/mutagent-evaluator/references/data-registry.md +43 -0
- package/.claude/skills/mutagent-evaluator/references/eval-layers.md +52 -0
- package/.claude/skills/mutagent-evaluator/references/eval-stage.md +1 -1
- package/.claude/skills/mutagent-evaluator/scripts/aggregate-discover.ts +254 -9
- package/.claude/skills/mutagent-evaluator/scripts/audience.ts +84 -0
- package/.claude/skills/mutagent-evaluator/scripts/code-eval-library.ts +201 -0
- package/.claude/skills/mutagent-evaluator/scripts/contracts/agentspec-evals.ts +101 -39
- package/.claude/skills/mutagent-evaluator/scripts/contracts/eval-matrix.ts +287 -3
- package/.claude/skills/mutagent-evaluator/scripts/contracts/eval-types.ts +30 -4
- package/.claude/skills/mutagent-evaluator/scripts/materialize-dataset.ts +130 -68
- package/.claude/skills/mutagent-evaluator/scripts/matrix-judge.ts +219 -1
- package/.claude/skills/mutagent-evaluator/scripts/merge-criteria.ts +849 -0
- package/.claude/skills/mutagent-evaluator/scripts/render-discover-report-v3.ts +1155 -0
- package/.claude/skills/mutagent-evaluator/scripts/render-eval-report-v3.ts +610 -0
- package/.claude/skills/mutagent-evaluator/scripts/render-eval-report.ts +2 -1
- package/.claude/skills/mutagent-evaluator/scripts/render-review-report-v3.ts +691 -0
- package/.claude/skills/mutagent-evaluator/scripts/route-failures.ts +10 -7
- package/.claude/skills/mutagent-evaluator/scripts/run-evaluate.ts +46 -1
- package/.claude/skills/mutagent-evaluator/scripts/run-review.ts +266 -0
- package/.claude/skills/mutagent-evaluator/scripts/verify-render.ts +728 -0
- package/package.json +2 -2
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="en"><head><meta charset="utf-8">
|
|
3
|
+
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
4
|
+
<link rel="icon" href="data:,">
|
|
5
|
+
<title>@@TITLE@@</title>
|
|
6
|
+
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
7
|
+
<link href="https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;500;600;700&family=IBM+Plex+Mono:wght@400;500;600&display=swap" rel="stylesheet">
|
|
8
|
+
<style>
|
|
9
|
+
:root{--bg:#0a0a12;--surf:#14141d;--surf2:#1a1a25;--surf3:#22222f;--border:rgba(255,255,255,.09);--borderS:rgba(255,255,255,.16);
|
|
10
|
+
--fg:#d6dbe6;--fgS:#eef1f6;--muted:#8a8698;--dim:#5a5667;--primary:#7E47D7;--primarySoft:#b794f4;--cyan:#45b8cc;
|
|
11
|
+
--pass:#43c39a;--fail:#e06666;--warn:#e8a64d;--pass-bg:rgba(67,195,154,.16);--fail-bg:rgba(224,102,102,.16);--warn-bg:rgba(232,166,77,.16);
|
|
12
|
+
--font:'Space Grotesk',system-ui,sans-serif;--mono:'IBM Plex Mono',ui-monospace,monospace;
|
|
13
|
+
--hm-pass:rgba(67,195,154,.55);--hm-fail:rgba(224,102,102,.6);--hm-indet:rgba(232,166,77,.6);--hm-skip:rgba(255,255,255,.16)}
|
|
14
|
+
*{box-sizing:border-box;border-radius:0}
|
|
15
|
+
body{margin:0;background:var(--bg);color:var(--fg);font-family:var(--font);font-size:14px;line-height:1.5;padding-bottom:110px}
|
|
16
|
+
.wrap{max-width:1180px;margin:0 auto;padding:0 22px 40px}
|
|
17
|
+
.hdr{position:sticky;top:0;z-index:30;background:rgba(10,10,18,.92);backdrop-filter:blur(8px);border-bottom:1px solid var(--border);padding:12px 22px;display:flex;align-items:center;gap:12px}
|
|
18
|
+
.wm{font-weight:700;font-size:18px;letter-spacing:.5px}
|
|
19
|
+
.wm b{background:linear-gradient(90deg,var(--primarySoft),var(--cyan));-webkit-background-clip:text;background-clip:text;color:transparent}
|
|
20
|
+
.mockbadge{font-family:var(--mono);font-size:10px;letter-spacing:.1em;color:var(--warn);border:1px solid var(--warn);padding:2px 8px}
|
|
21
|
+
.panel{border:1px solid var(--border);background:var(--surf);margin-top:14px}
|
|
22
|
+
.panel-h{display:flex;align-items:center;gap:8px;padding:9px 15px;border-bottom:1px solid var(--border);font-family:var(--mono);font-size:11px;text-transform:uppercase;letter-spacing:.06em;color:var(--primarySoft);background:var(--surf2)}
|
|
23
|
+
.panel-h .r{margin-left:auto;color:var(--muted);text-transform:none}
|
|
24
|
+
.panel-b{padding:13px 15px}
|
|
25
|
+
.tabs{display:flex;border-bottom:1px solid var(--border);background:var(--bg);flex-wrap:wrap}
|
|
26
|
+
.tabs s{font-family:var(--mono);font-size:11px;padding:8px 13px;border-right:1px solid var(--border);color:var(--muted);cursor:pointer;text-decoration:none}
|
|
27
|
+
.tabs s.on{color:var(--primarySoft);border-bottom:2px solid var(--primary);background:var(--surf)}
|
|
28
|
+
.sec{display:none}.sec.show{display:block}
|
|
29
|
+
.kpi{display:flex;gap:7px;flex-wrap:wrap}
|
|
30
|
+
.kpi i{font-family:var(--mono);font-size:11px;font-style:normal;padding:3px 9px;border:1px solid var(--border);color:var(--muted)}
|
|
31
|
+
.kpi i b{color:var(--fgS)} .kpi i.pass b{color:var(--pass)} .kpi i.fail b{color:var(--fail)} .kpi i.warn b{color:var(--warn)} .kpi i.p b{color:var(--primarySoft)}
|
|
32
|
+
.sev{font-family:var(--mono);font-size:11px;font-weight:700;padding:2px 7px}
|
|
33
|
+
.sev.CRIT{color:var(--fail);background:var(--fail-bg)} .sev.HIGH{color:var(--warn);background:var(--warn-bg)} .sev.MED{color:var(--cyan);background:rgba(69,184,204,.10)} .sev.LOW{color:var(--muted);background:var(--surf3)}
|
|
34
|
+
.chip{font-family:var(--mono);font-size:11px;color:var(--muted);border:1px solid var(--border);padding:1px 6px}
|
|
35
|
+
.gb{font-family:var(--mono);font-size:11px;padding:2px 7px}
|
|
36
|
+
.gb.observed{color:var(--pass);background:var(--pass-bg)} .gb.inferred{color:var(--warn);background:var(--warn-bg)}
|
|
37
|
+
/* funnel variants */
|
|
38
|
+
.funnelH{display:flex;gap:0;align-items:stretch;margin:8px 0;overflow-x:auto}
|
|
39
|
+
.funnelH .st{flex:1;min-width:130px;border:1px solid var(--border);background:var(--surf2);padding:9px 12px;position:relative}
|
|
40
|
+
.funnelH .st:not(:last-child)::after{content:"▸";position:absolute;right:-9px;top:38%;color:var(--primarySoft);z-index:2}
|
|
41
|
+
.funnelH .st .k{font-family:var(--mono);font-size:11px;letter-spacing:.08em;text-transform:uppercase;color:var(--muted)}
|
|
42
|
+
.funnelH .st .v{font-size:22px;font-weight:700;color:var(--fgS)}
|
|
43
|
+
.funnelH .st .s{font-size:12.5px;color:var(--muted)}
|
|
44
|
+
.funnelV{margin:8px 0;border-left:2px solid var(--primary);padding-left:16px}
|
|
45
|
+
.funnelV .st{margin:0 0 10px;position:relative}
|
|
46
|
+
.funnelV .st::before{content:"";position:absolute;left:-21px;top:6px;width:8px;height:8px;background:var(--primary)}
|
|
47
|
+
.funnelV .st .v{font-weight:700;color:var(--fgS)}
|
|
48
|
+
.funnelV .st .s{font-family:var(--mono);font-size:11px;color:var(--muted)}
|
|
49
|
+
/* bars / curve */
|
|
50
|
+
.bar{display:flex;align-items:center;gap:10px;font-family:var(--mono);font-size:13px;margin:6px 0}
|
|
51
|
+
.bar .lab{width:200px;color:var(--fg);text-align:left;font-weight:600}
|
|
52
|
+
.bar .tr{flex:1;background:var(--surf3);height:16px;position:relative}
|
|
53
|
+
.bar .fill{height:16px;background:rgba(126,71,215,.55)}
|
|
54
|
+
.bar .fill.g{background:rgba(67,195,154,.5)} .bar .fill.r{background:rgba(224,102,102,.5)} .bar .fill.c{background:rgba(69,184,204,.5)}
|
|
55
|
+
.bar b{color:var(--fgS);min-width:64px;font-size:14px}
|
|
56
|
+
/* section intro — replaces wall-of-text explainers */
|
|
57
|
+
.intro{margin:0 0 16px}
|
|
58
|
+
.intro h4{margin:0 0 6px;font-size:16px;color:var(--fgS);font-weight:600}
|
|
59
|
+
.intro p{margin:0 0 6px;font-size:14px;color:var(--muted);max-width:84ch;line-height:1.6}
|
|
60
|
+
.intro p b{color:var(--fg)}
|
|
61
|
+
.h4s{margin:20px 0 8px;font-size:15px;color:var(--fgS);font-weight:600;border-bottom:1px solid var(--border);padding-bottom:5px;max-width:84ch}
|
|
62
|
+
.stratexp{font-size:13.5px;color:var(--muted);margin:-2px 0 14px 0;max-width:80ch;line-height:1.55}
|
|
63
|
+
.stratexp b{color:var(--fg)}
|
|
64
|
+
/* sampled-corpus heat grid */
|
|
65
|
+
.heatwrap{margin:14px 0 4px}
|
|
66
|
+
.heatgrid{display:grid;grid-template-columns:repeat(43,1fr);gap:2px;max-width:860px}
|
|
67
|
+
.hg{aspect-ratio:1;background:var(--surf3);border:1px solid rgba(0,0,0,.3)}
|
|
68
|
+
.hg.samp{background:rgba(126,71,215,.45)}
|
|
69
|
+
.hg.readp{background:var(--hm-pass, rgba(67,195,154,.55));background:rgba(67,195,154,.55)}
|
|
70
|
+
.hg.readf{background:rgba(224,102,102,.6)}
|
|
71
|
+
.hlegend{display:flex;gap:16px;flex-wrap:wrap;font-family:var(--mono);font-size:12px;color:var(--muted);margin-top:8px}
|
|
72
|
+
.hlegend .sw{display:inline-block;width:11px;height:11px;vertical-align:-1px;margin-right:5px;border:1px solid rgba(0,0,0,.3)}
|
|
73
|
+
/* criterion cards vs table */
|
|
74
|
+
.ccard{border:1px solid var(--border);background:var(--surf2);margin:10px 0}
|
|
75
|
+
.ccard-h{display:flex;gap:9px;align-items:center;padding:8px 11px;border-bottom:1px solid var(--border);flex-wrap:wrap}
|
|
76
|
+
.ccard-b{padding:10px 13px;font-size:14px;color:var(--muted);line-height:1.55}
|
|
77
|
+
.ccard-b b{color:var(--fg)}
|
|
78
|
+
.nest{border:1px solid var(--border);background:var(--surf);margin:8px 0;padding:7px 10px;font-family:var(--mono);font-size:13px;color:var(--muted);line-height:1.5}
|
|
79
|
+
.nest b{color:var(--fg)} .nest .ok{color:var(--pass)}
|
|
80
|
+
table.ct{border-collapse:collapse;width:100%;font-size:14px;margin:8px 0}
|
|
81
|
+
table.ct th{font-family:var(--mono);font-size:11px;letter-spacing:.06em;text-transform:uppercase;color:var(--muted);text-align:left;padding:7px 10px;border-bottom:1px solid var(--borderS)}
|
|
82
|
+
table.ct td{padding:8px 10px;border-bottom:1px solid var(--border);color:var(--muted);vertical-align:top;line-height:1.55}
|
|
83
|
+
table.ct td b{color:var(--fg)}
|
|
84
|
+
.vq{border:1px dashed var(--cyan);background:rgba(69,184,204,.05);padding:8px 11px;margin:12px 0}
|
|
85
|
+
.vq .q{font-family:var(--mono);font-size:10px;letter-spacing:.08em;text-transform:uppercase;color:var(--cyan);margin-bottom:5px}
|
|
86
|
+
.vq .opts{display:flex;gap:6px;flex-wrap:wrap}
|
|
87
|
+
.vq .opts button{font-family:var(--mono);font-size:11px;padding:3px 11px;background:var(--surf2);border:1px solid var(--border);color:var(--muted);cursor:pointer}
|
|
88
|
+
.vq .opts button.on{border-color:var(--cyan);color:var(--cyan);background:rgba(69,184,204,.12)}
|
|
89
|
+
.row{font-family:var(--mono);font-size:13.5px;color:var(--fg);opacity:.92;margin:5px 0;line-height:1.55}
|
|
90
|
+
.row .ok{color:var(--pass)} .row .w{color:var(--warn)}
|
|
91
|
+
.explain{font-size:14px;line-height:1.6;color:var(--muted);border-left:3px solid var(--cyan);background:rgba(69,184,204,.05);padding:8px 13px;margin:0 0 14px;max-width:86ch}
|
|
92
|
+
.explain b{color:var(--fg)}
|
|
93
|
+
/* criterion card v2 — labeled sections, show-don't-tell */
|
|
94
|
+
.csec{padding:10px 14px 11px;border-bottom:1px solid var(--border)}
|
|
95
|
+
.csec:last-child{border-bottom:0}
|
|
96
|
+
.clab{font-family:var(--mono);font-size:12px;font-weight:600;letter-spacing:.03em;color:var(--primarySoft);margin-bottom:5px}
|
|
97
|
+
.dotstrip{display:inline-flex;gap:3px;vertical-align:-2px;margin:0 8px}
|
|
98
|
+
.dotstrip u{width:13px;height:13px;border:1px solid rgba(0,0,0,.35)}
|
|
99
|
+
.dotstrip u.fail{background:var(--hm-fail)} .dotstrip u.pass{background:var(--hm-pass)} .dotstrip u.uncertain{background:var(--hm-indet)} .dotstrip u.skip{background:var(--hm-skip)}
|
|
100
|
+
.origin{font-family:var(--mono);font-size:12.5px;color:var(--primarySoft);border:1px solid var(--primary);padding:1px 8px;cursor:pointer;margin:2px 4px 2px 0;display:inline-block}
|
|
101
|
+
.origin:hover{background:rgba(126,71,215,.18)}
|
|
102
|
+
.quote{font-family:var(--mono);font-size:13px;line-height:1.55;color:var(--fg);background:var(--bg);border:1px solid var(--border);border-left:3px solid var(--pass);padding:4px 9px;margin:4px 0;display:block}
|
|
103
|
+
.quote i{font-style:normal;color:var(--dim)}
|
|
104
|
+
#tpeek{display:none;position:fixed;z-index:70;background:var(--surf3);border:1px solid var(--primary);padding:11px 14px;font-size:13.5px;color:var(--fg);max-width:460px;box-shadow:0 8px 26px rgba(0,0,0,.55);line-height:1.55}
|
|
105
|
+
#tpeek .pk1{font-family:var(--mono);font-weight:700;color:var(--fgS);font-size:14px;margin-bottom:4px;padding-right:66px}
|
|
106
|
+
#tpeek .pkstep{font-family:var(--mono);font-size:12.5px;color:var(--muted);padding:2px 0;border-bottom:1px dashed var(--border)}
|
|
107
|
+
#tpeek .pkstep b{color:var(--cyan);font-weight:600}
|
|
108
|
+
#tpeek .pkx{position:absolute;top:6px;right:9px;color:var(--dim);cursor:pointer;font-family:var(--mono)}
|
|
109
|
+
.handoffbtn{background:var(--surf2);border:1px solid var(--pass);color:var(--pass);font-family:var(--mono);font-size:11px;letter-spacing:.05em;text-transform:none;padding:4px 12px;cursor:pointer;margin-left:auto}
|
|
110
|
+
.handoffbtn.done{background:var(--pass-bg)}
|
|
111
|
+
.layercover{display:flex;gap:8px;flex-wrap:wrap;margin:10px 0 4px}
|
|
112
|
+
.layercover i{font-style:normal;font-family:var(--mono);font-size:12.5px;padding:3px 10px;border:1px solid var(--border);color:var(--muted)}
|
|
113
|
+
.layercover i b{color:var(--fgS)}
|
|
114
|
+
.stratline{font-size:13.5px;color:var(--muted);margin:-2px 0 10px 200px;max-width:72ch}
|
|
115
|
+
.hero{padding:26px 0 10px;border-bottom:1px solid var(--border);margin-bottom:6px}
|
|
116
|
+
.hero .who{font-family:var(--mono);font-size:11.5px;color:var(--cyan);text-transform:uppercase;letter-spacing:.14em;margin-bottom:7px}
|
|
117
|
+
.hero h1{font-size:clamp(24px,3.4vw,32px);margin:0 0 8px;color:var(--fgS);letter-spacing:-.01em;font-weight:700}
|
|
118
|
+
.hero p{font-size:15.5px;color:var(--muted);max-width:88ch;margin:0 0 8px;line-height:1.65}
|
|
119
|
+
.hero p b{color:var(--fg)}
|
|
120
|
+
.foot{margin-top:32px;color:var(--dim);font-size:11px;border-top:1px solid var(--border);padding-top:12px;font-family:var(--mono)}
|
|
121
|
+
#fbpanel{position:fixed;bottom:0;left:0;right:0;background:rgba(10,10,18,.97);border-top:1px solid var(--primary);padding:9px 22px 11px;z-index:50}
|
|
122
|
+
#fbpanel .frow{max-width:1180px;margin:0 auto;display:flex;gap:12px;align-items:flex-end}
|
|
123
|
+
#fbpanel label{font-family:var(--mono);font-size:10px;letter-spacing:.1em;text-transform:uppercase;color:var(--primarySoft);display:block;margin-bottom:4px}
|
|
124
|
+
#fbtext{flex:1;background:var(--surf2);border:1px solid var(--border);color:var(--fg);font-family:var(--font);font-size:13px;padding:7px 10px;min-height:40px;max-height:110px;resize:vertical}
|
|
125
|
+
#fbcopy{background:var(--surf2);border:1px solid var(--primary);color:var(--primarySoft);font-family:var(--mono);font-size:11px;letter-spacing:.08em;text-transform:uppercase;padding:10px 16px;cursor:pointer;white-space:nowrap}
|
|
126
|
+
#fbcopy.done{border-color:var(--pass);color:var(--pass)}
|
|
127
|
+
</style></head><body>
|
|
128
|
+
<div class="hdr"><div class="wm">MUTAGENT<b>/evaluator</b> · Discovery Report</div>
|
|
129
|
+
@@HEADER_RIGHT@@</div>
|
|
130
|
+
<div class="wrap">
|
|
131
|
+
|
|
132
|
+
<div class="hero" data-component="mgt-report-hero">
|
|
133
|
+
<div class="who">Eval discovery · which checks should exist</div>
|
|
134
|
+
<h1>@@HERO_TITLE@@</h1>
|
|
135
|
+
<p>@@HERO_LEDE@@</p>
|
|
136
|
+
</div>
|
|
137
|
+
|
|
138
|
+
<div class="panel"><div class="panel-b" style="display:flex;align-items:center;gap:10px;flex-wrap:wrap">
|
|
139
|
+
@@IDENTITY_STRIP@@
|
|
140
|
+
</div></div>
|
|
141
|
+
|
|
142
|
+
<div class="panel" id="report">
|
|
143
|
+
<div class="tabs">
|
|
144
|
+
<s class="on" onclick="tab(0)">① Overview</s><s onclick="tab(1)">② Corpus & Sampling</s>
|
|
145
|
+
<s onclick="tab(2)">③ Discovery Walk</s><s onclick="tab(3)">④ Criteria Catalog</s>
|
|
146
|
+
<s onclick="tab(4)">⑤ Dataset</s><s onclick="tab(5)">⑥ Methodology [INT]</s>
|
|
147
|
+
</div>
|
|
148
|
+
|
|
149
|
+
<!-- ① OVERVIEW -->
|
|
150
|
+
<div class="sec show" data-page="overview"><div class="panel-h">① Overview <span class="r">the funnel IS the story</span></div><div class="panel-b">
|
|
151
|
+
<div class="intro">
|
|
152
|
+
<h4>What this run did</h4>
|
|
153
|
+
<p>@@OVERVIEW_INTRO@@</p>
|
|
154
|
+
</div>
|
|
155
|
+
<div id="funnelhost" data-component="mgt-funnel"></div>
|
|
156
|
+
|
|
157
|
+
<div class="heatwrap" data-component="mgt-sample-heatgrid">
|
|
158
|
+
<div class="h4s" style="margin-top:18px">@@HEAT_HEADING@@</div>
|
|
159
|
+
<div class="hlegend" style="margin:0 0 10px;font-size:13px">@@HEAT_LEGEND@@</div>
|
|
160
|
+
<div class="heatgrid" id="heatgrid"></div>
|
|
161
|
+
</div>
|
|
162
|
+
|
|
163
|
+
<div class="kpi" style="margin-top:16px">
|
|
164
|
+
@@KPI@@
|
|
165
|
+
</div>
|
|
166
|
+
<div style="margin-top:12px;color:var(--muted);font-size:14px;max-width:84ch;line-height:1.6">
|
|
167
|
+
@@BOTTOM_LINE@@
|
|
168
|
+
</div>
|
|
169
|
+
</div></div>
|
|
170
|
+
|
|
171
|
+
<!-- ② CORPUS -->
|
|
172
|
+
<div class="sec" data-page="corpus-sampling"><div class="panel-h">② Corpus & Sampling <span class="r">where the traces came from · how we chose · when we stopped</span></div><div class="panel-b">
|
|
173
|
+
|
|
174
|
+
@@SAMPLING_SECTION@@
|
|
175
|
+
<div class="nest" style="margin-top:12px">Data-hygiene guard: tool observations are sorted by real event time before any "was X known at that moment" judgment — unsorted platform exports flip most such verdicts.</div>
|
|
176
|
+
</div></div>
|
|
177
|
+
|
|
178
|
+
<!-- ③ DISCOVERY WALK -->
|
|
179
|
+
<div class="sec" data-page="discovery-walk"><div class="panel-h">③ Discovery Walk <span class="r">failure classes → emergent clusters → criteria</span></div><div class="panel-b">
|
|
180
|
+
<div class="intro">
|
|
181
|
+
<h4>How each failure was diagnosed</h4>
|
|
182
|
+
<p>Every failed session was read for its <b>first thing wrong</b>, and each finding was classified
|
|
183
|
+
<b>behavioral</b> (an eval belongs here) or <b>infrastructure</b> (a fix belongs elsewhere). Failures were then
|
|
184
|
+
grouped into categories that <b>emerged from the reading itself</b> — never from a pre-made list,
|
|
185
|
+
which would bias what we find. Each category ended one of three ways: it became an eval criterion, it was routed to
|
|
186
|
+
diagnostics as an infrastructure fix, or it was dropped as a one-off.</p>
|
|
187
|
+
</div>
|
|
188
|
+
@@WALK_SECTION@@
|
|
189
|
+
|
|
190
|
+
</div></div>
|
|
191
|
+
|
|
192
|
+
<!-- ④ CATALOG -->
|
|
193
|
+
<div class="sec" data-page="criteria-catalog"><div class="panel-h">④ Criteria Catalog <span class="r">@@CATALOG_COUNT@@</span><button class="handoffbtn" id="ahbtn" onclick="copyAdoptionHandoff()">Copy adoption handoff (markdown)</button></div><div class="panel-b">
|
|
194
|
+
<div class="intro" style="margin-top:2px">
|
|
195
|
+
<h4>How to read a criterion card</h4>
|
|
196
|
+
<p>Each card is one recommended check. <b>THE CHECK</b> is what gets verified — true or false per session, never a score.
|
|
197
|
+
<b>WHY IT EXISTS</b> shows the real past failures that created it: the colored strip is every analyzed session
|
|
198
|
+
(<b style="color:var(--fail)">red = this check failed there</b>), and the purple session chips are clickable — peek at the
|
|
199
|
+
actual trace. <b>EVIDENCE SEEN</b> quotes the exact strings the verdicts cited. <b>GATES RELEASE?</b> says whether a
|
|
200
|
+
failure of this check blocks a release.</p>
|
|
201
|
+
</div>
|
|
202
|
+
<div class="layercover" data-component="mgt-layer-coverage">
|
|
203
|
+
@@LAYER_COVERAGE@@
|
|
204
|
+
</div>
|
|
205
|
+
<div id="cathost" data-component="mgt-criterion-catalog"></div>
|
|
206
|
+
</div></div>
|
|
207
|
+
|
|
208
|
+
<!-- ⑤ DATASET -->
|
|
209
|
+
<div class="sec" data-page="dataset"><div class="panel-h">⑤ Dataset <span class="r">candidates → *build-dataset · living-suite Δ</span></div><div class="panel-b">
|
|
210
|
+
<div class="intro">
|
|
211
|
+
<h4>Proposed regression cases — keep these mistakes fixed forever</h4>
|
|
212
|
+
<p>Some sessions showed failure shapes worth <b>keeping as test cases</b>: each becomes a synthetic input that recreates the
|
|
213
|
+
situation, so every future evaluation automatically re-checks that this exact mistake hasn't come back. Each row tells you
|
|
214
|
+
the real story it came from, how that session was judged, and what the test case will prevent.</p>
|
|
215
|
+
</div>
|
|
216
|
+
@@DATASET_SECTION@@
|
|
217
|
+
</div></div>
|
|
218
|
+
|
|
219
|
+
<!-- ⑥ METHODOLOGY -->
|
|
220
|
+
<div class="sec" data-page="methodology"><div class="panel-h">⑥ Methodology <span class="r">internal — stripped on publish</span></div><div class="panel-b">
|
|
221
|
+
<div class="intro">
|
|
222
|
+
<h4>The report grading itself</h4>
|
|
223
|
+
<p>The full account of HOW this run produced its numbers: the process as it actually executed, which script or agent
|
|
224
|
+
produced every figure on the other tabs (provenance), one fully worked example of a session read, and the honest gaps.
|
|
225
|
+
Internal only — stripped from any external publish.</p>
|
|
226
|
+
</div>
|
|
227
|
+
|
|
228
|
+
@@METHODOLOGY_SECTION@@
|
|
229
|
+
</div></div>
|
|
230
|
+
</div>
|
|
231
|
+
|
|
232
|
+
<div class="foot">@@FOOT@@</div>
|
|
233
|
+
</div>
|
|
234
|
+
|
|
235
|
+
<div id="fbpanel" data-component="mgt-feedback-bar"><div class="frow">
|
|
236
|
+
<div style="flex:1"><label for="fbtext">@@FEEDBACK_LABEL@@</label>
|
|
237
|
+
<textarea id="fbtext" placeholder="e.g. 'funnel horizontal but add drop-off %' · 'catalog: show passDef/failDef on cards' …"></textarea></div>
|
|
238
|
+
<button id="fbcopy" type="button">Copy MD</button>
|
|
239
|
+
</div></div>
|
|
240
|
+
<div id="tpeek" data-component="mgt-trace-peek"></div>
|
|
241
|
+
|
|
242
|
+
<script>window.ASTER=@@DATA_JSON@@;</script>
|
|
243
|
+
<script>
|
|
244
|
+
function tab(i){var t=document.querySelectorAll('#report>.tabs s'),s=document.querySelectorAll('#report>.sec');
|
|
245
|
+
t.forEach(function(x,k){x.classList.toggle('on',k===i)});s.forEach(function(x,k){x.classList.toggle('show',k===i)});}
|
|
246
|
+
var funnelStyle='h', catStyle='cards';
|
|
247
|
+
var STAGES=ASTER.funnelStages||[];
|
|
248
|
+
function renderFunnel(){
|
|
249
|
+
var host=document.getElementById('funnelhost'),h='';
|
|
250
|
+
if(funnelStyle==='h'){
|
|
251
|
+
h='<div class="funnelH">';
|
|
252
|
+
STAGES.forEach(function(s){h+='<div class="st"><div class="k">'+s[0]+'</div><div class="v">'+s[1]+'</div><div class="s">'+s[2]+'</div></div>';});
|
|
253
|
+
h+='</div>';
|
|
254
|
+
}else{
|
|
255
|
+
h='<div class="funnelV">';
|
|
256
|
+
STAGES.forEach(function(s){h+='<div class="st"><span class="v">'+s[1]+'</span> <b style="color:var(--primarySoft)">'+s[0]+'</b> <span class="s">— '+s[2]+'</span></div>';});
|
|
257
|
+
h+='</div>';
|
|
258
|
+
}
|
|
259
|
+
host.innerHTML=h;
|
|
260
|
+
}
|
|
261
|
+
var CLUSTERS=ASTER.clusters||{};
|
|
262
|
+
function critEvidence(cid){
|
|
263
|
+
var out={cells:[],origins:[],quotes:[]};
|
|
264
|
+
ASTER.traces.forEach(function(tr){
|
|
265
|
+
var st='pass',ref=null,vseen=null;
|
|
266
|
+
tr.steps.forEach(function(s){(s.verdicts||[]).forEach(function(vd){if(vd.crit===cid){vseen=vd.verdict;if(vd.ref)ref=vd.ref;}});});
|
|
267
|
+
if(!vseen){st=(tr.gate==='incomplete')?'skip':'pass';}
|
|
268
|
+
else st=(vseen==='uncertain')?'uncertain':vseen;
|
|
269
|
+
out.cells.push({id:tr.id,st:st});
|
|
270
|
+
if(vseen==='fail'||vseen==='uncertain'){out.origins.push(tr.id);if(ref)out.quotes.push({id:tr.id,ref:ref});}
|
|
271
|
+
});
|
|
272
|
+
return out;
|
|
273
|
+
}
|
|
274
|
+
function renderCat(){
|
|
275
|
+
var host=document.getElementById('cathost'),h='';
|
|
276
|
+
if(catStyle==='cards'){
|
|
277
|
+
ASTER.criteria.forEach(function(c){
|
|
278
|
+
var ev=critEvidence(c.id);
|
|
279
|
+
var dots;
|
|
280
|
+
if(ev.cells.length<=60){
|
|
281
|
+
dots='<span class="dotstrip">';
|
|
282
|
+
ev.cells.forEach(function(cl){dots+='<u class="'+cl.st+'" title="'+cl.id+': '+cl.st+'"></u>';});
|
|
283
|
+
dots+='</span>';
|
|
284
|
+
}else{
|
|
285
|
+
/* mgt-ratio-bar — large-corpus aggregate of the per-session dot strip */
|
|
286
|
+
var f=0,u=0;ev.cells.forEach(function(cl){if(cl.st==='fail')f++;else if(cl.st==='uncertain')u++;});
|
|
287
|
+
var n=ev.cells.length,fp=Math.max(1,Math.round(100*f/n)),up=Math.round(100*u/n);
|
|
288
|
+
dots='<span data-component="mgt-ratio-bar" style="display:inline-flex;align-items:center;gap:8px;vertical-align:-3px;margin:0 8px">'
|
|
289
|
+
+'<span style="display:inline-flex;width:180px;height:14px;border:1px solid rgba(0,0,0,.35);background:var(--hm-pass, rgba(67,195,154,.38));background:rgba(67,195,154,.38)">'
|
|
290
|
+
+'<span style="width:'+fp+'%;background:rgba(224,102,102,.7)"></span>'
|
|
291
|
+
+(u?'<span style="width:'+up+'%;background:rgba(232,166,77,.6)"></span>':'')
|
|
292
|
+
+'</span><b style="color:var(--fg);font-family:var(--mono);font-size:13px">failed in '+f+' of '+n+'</b></span>';
|
|
293
|
+
}
|
|
294
|
+
var shown=ev.origins.slice(0,8);
|
|
295
|
+
var origins='';shown.forEach(function(id){origins+='<span class="origin" onclick="peekTrace(event,\''+id+'\')">'+id+' ▸ peek</span>';});
|
|
296
|
+
if(ev.origins.length>8)origins+='<span class="chip" style="font-size:12.5px">+'+(ev.origins.length-8)+' more sessions</span>';
|
|
297
|
+
var quotes='';ev.quotes.forEach(function(q){quotes+='<span class="quote">"'+q.ref.value+'" <i>— </i><span class="origin" onclick="peekTrace(event,\''+q.id+'\')" style="font-size:11.5px">'+q.id+(q.ref.obs&&q.ref.obs!==q.id?' · '+q.ref.obs:'')+' · '+q.ref.path+' ▸ peek</span></span>';});
|
|
298
|
+
if(!quotes)quotes='<span class="quote" style="border-left-color:var(--warn)">no quoted evidence for this check in this sample — no failed session cited a structured ref, so this criterion is INFERRED rather than observed</span>';
|
|
299
|
+
h+='<div class="ccard" data-component="mgt-criterion-card"><div class="ccard-h"><span class="sev '+c.sev+'">'+c.sev+'</span><b style="font-size:15px">'+c.id+' · '+c.slug+'</b>'
|
|
300
|
+
+'<span class="chip" style="color:var(--primarySoft);border-color:rgba(126,71,215,.5)">layer: '+c.layer+'</span>'
|
|
301
|
+
+'<span class="chip">'+(c.cls==='code'?'checked by code':c.cls==='hybrid'?'checked by code + judge':'checked by judge')+'</span></div>'
|
|
302
|
+
+'<div class="csec"><div class="clab">The check</div><div style="font-size:14.5px;color:var(--fg)">'+c.statement+'</div>'
|
|
303
|
+
+'<div style="font-size:13px;color:var(--muted);margin-top:4px"><b style="color:var(--fg)">passes:</b> '+c.passDef+' · <b style="color:var(--fg)">fails:</b> '+c.failDef+'</div></div>'
|
|
304
|
+
+'<div class="csec"><div class="clab">Why it exists — the real failures that created it</div>'
|
|
305
|
+
+'<div style="font-size:14px;color:var(--muted)">Cluster: <b style="color:var(--fg)">'+CLUSTERS[c.id]+'</b> · behavior across the '+ev.cells.length+' analyzed session(s): '+dots
|
|
306
|
+
+(ev.origins.length?'<br>failed / undecided in: '+origins:' — no failure in this sample')+'</div></div>'
|
|
307
|
+
+'<div class="csec"><div class="clab">Evidence seen — exact quoted strings</div>'+quotes+'</div>'
|
|
308
|
+
+'<div class="csec"><div class="clab">Does it gate a release?</div><div style="font-size:14px;color:var(--muted)">'
|
|
309
|
+
+(c.sev==='CRIT'||c.sev==='HIGH'?'<b style="color:var(--fail)">YES</b> — a '+c.sev+' failure fails the gate':'<b style="color:var(--muted)">no</b> — advisory, tracked in the variance view')
|
|
310
|
+
+'</div></div></div>';
|
|
311
|
+
});
|
|
312
|
+
}else{
|
|
313
|
+
h='<table class="ct"><tr><th>id</th><th>statement</th><th>layer</th><th>class</th><th>sev</th><th>grounding</th></tr>';
|
|
314
|
+
ASTER.criteria.forEach(function(c){
|
|
315
|
+
h+='<tr><td><b>'+c.id+'</b></td><td>'+c.statement+'</td><td>'+c.layer+'</td><td>'+c.cls+'</td>'
|
|
316
|
+
+'<td><span class="sev '+c.sev+'">'+c.sev+'</span></td><td><span class="gb observed">obs k='+c.k+'/'+c.n+'</span></td></tr>';
|
|
317
|
+
});
|
|
318
|
+
h+='</table>';
|
|
319
|
+
}
|
|
320
|
+
host.innerHTML=h;
|
|
321
|
+
}
|
|
322
|
+
function renderHeat(){
|
|
323
|
+
var host=document.getElementById('heatgrid');if(!host)return;
|
|
324
|
+
var H=ASTER.heat||{readp:0,readf:0,samp:0};
|
|
325
|
+
var cells=[];
|
|
326
|
+
for(var i=0;i<H.readp;i++)cells.push('readp');
|
|
327
|
+
for(var j=0;j<H.readf;j++)cells.push('readf');
|
|
328
|
+
for(var k=0;k<H.samp;k++)cells.push('samp');
|
|
329
|
+
/* deterministic interleave so colors mix like a real corpus (no Math.random — stable render) */
|
|
330
|
+
var total=cells.length,readTotal=H.readp+H.readf;
|
|
331
|
+
var mixed=[],a=0,b=H.readp,c=readTotal,step=0;
|
|
332
|
+
while(mixed.length<total){
|
|
333
|
+
if(step%3===0&&a<H.readp)mixed.push(cells[a++]);
|
|
334
|
+
else if(step%3===1&&b<readTotal)mixed.push(cells[b++]);
|
|
335
|
+
else if(c<total)mixed.push(cells[c++]);
|
|
336
|
+
else if(a<H.readp)mixed.push(cells[a++]);
|
|
337
|
+
else if(b<readTotal)mixed.push(cells[b++]);
|
|
338
|
+
step++;
|
|
339
|
+
}
|
|
340
|
+
var h='';mixed.forEach(function(cl){h+='<span class="hg '+cl+'"></span>';});
|
|
341
|
+
host.innerHTML=h;
|
|
342
|
+
}
|
|
343
|
+
function copyAdoptionHandoff(){
|
|
344
|
+
var md=ASTER.adoptionMd||'';var btn=document.getElementById('ahbtn');
|
|
345
|
+
function ok(){btn.classList.add('done');btn.textContent='✓ copied';setTimeout(function(){btn.classList.remove('done');btn.textContent='Copy adoption handoff (markdown)';},2000);}
|
|
346
|
+
if(navigator.clipboard&&navigator.clipboard.writeText){navigator.clipboard.writeText(md).then(ok,function(){fbc(md);});}else{fbc(md);}
|
|
347
|
+
function fbc(t){var x=document.createElement('textarea');x.value=t;document.body.appendChild(x);x.select();
|
|
348
|
+
try{document.execCommand('copy');}catch(e){}document.body.removeChild(x);ok();}
|
|
349
|
+
}
|
|
350
|
+
function peekTrace(e,id){
|
|
351
|
+
e.stopPropagation();
|
|
352
|
+
var tr=null;ASTER.traces.forEach(function(t){if(t.id===id)tr=t;});
|
|
353
|
+
var p=document.getElementById('tpeek');
|
|
354
|
+
if(!tr){p.style.display='none';return;}
|
|
355
|
+
var h='<span class="pkx" onclick="document.getElementById(\'tpeek\').style.display=\'none\'">✕ close</span>'
|
|
356
|
+
+'<div class="pk1">'+tr.id+' · '+tr.scenario+' · gate '+tr.gate+'</div>'
|
|
357
|
+
+'<div style="font-size:13px;color:var(--muted);margin-bottom:6px">'+(tr.outcome.verdict==='pass'?('reached expected exit: '+tr.outcome.expected):('expected '+tr.outcome.expected+' — actual: '+tr.outcome.actual))+'</div>';
|
|
358
|
+
tr.steps.forEach(function(s,i){h+='<div class="pkstep">obs-'+(i+1)+' · <b>'+s.tool+'</b> '+String(s.args)+'<br><span style="color:var(--fg);opacity:.85">→ '+String(s.output).slice(0,400)+'</span></div>';});
|
|
359
|
+
if(tr.divergence)h+='<div style="font-size:12.5px;color:var(--fail);margin-top:5px">◆ '+tr.divergence+'</div>';
|
|
360
|
+
if(ASTER.evalReportHref){h+='<div style="font-size:12.5px;margin-top:8px"><a href="'+ASTER.evalReportHref+'#'+tr.id+'" target="_blank" style="color:var(--primarySoft);text-decoration:none;border:1px solid var(--primary);padding:2px 9px;font-family:var(--mono)">open the full judged walk ('+tr.id+') in the Evaluation Report ▸</a></div>';}
|
|
361
|
+
p.innerHTML=h;p.style.display='block';
|
|
362
|
+
var x=e.clientX+14,y=e.clientY+14;
|
|
363
|
+
if(x+470>window.innerWidth)x=Math.max(10,e.clientX-480);
|
|
364
|
+
if(y+320>window.innerHeight)y=Math.max(10,e.clientY-330);
|
|
365
|
+
p.style.left=x+'px';p.style.top=y+'px';
|
|
366
|
+
}
|
|
367
|
+
document.addEventListener('click',function(e){
|
|
368
|
+
var p=document.getElementById('tpeek');
|
|
369
|
+
if(p&&p.style.display==='block'&&!p.contains(e.target)&&!(e.target.classList&&e.target.classList.contains('origin')))p.style.display='none';
|
|
370
|
+
});
|
|
371
|
+
function setFunnel(v,btn){funnelStyle=v;vqOn(btn);renderFunnel();}
|
|
372
|
+
function setCat(v,btn){catStyle=v;vqOn(btn);renderCat();}
|
|
373
|
+
function vqOn(btn){btn.parentElement.querySelectorAll('button').forEach(function(b){b.classList.remove('on');});btn.classList.add('on');}
|
|
374
|
+
renderFunnel();renderCat();renderHeat();
|
|
375
|
+
(function(){
|
|
376
|
+
var btn=document.getElementById('fbcopy'),ta=document.getElementById('fbtext');
|
|
377
|
+
if(!btn||!ta)return;
|
|
378
|
+
function buildMd(){
|
|
379
|
+
var lines=['## Feedback — '+(ASTER.feedbackTitle||document.title),'','Artifact: `'+(ASTER.artifactPath||'discovery-report.html')+'`'];
|
|
380
|
+
document.querySelectorAll('.vq').forEach(function(vq){
|
|
381
|
+
var on=vq.querySelector('button.on');
|
|
382
|
+
lines.push('- **'+vq.getAttribute('data-vq')+'**: '+(on?on.getAttribute('data-v'):'(none)'));
|
|
383
|
+
});
|
|
384
|
+
lines.push('','### Notes');
|
|
385
|
+
var note=ta.value.trim();lines.push(note.length?note:'_(no notes entered)_');
|
|
386
|
+
return lines.join('\n');
|
|
387
|
+
}
|
|
388
|
+
function done(){btn.classList.add('done');btn.textContent='Copied ✓';setTimeout(function(){btn.classList.remove('done');btn.textContent='Copy MD';},1800);}
|
|
389
|
+
btn.addEventListener('click',function(){
|
|
390
|
+
var md=buildMd();
|
|
391
|
+
if(navigator.clipboard&&navigator.clipboard.writeText){navigator.clipboard.writeText(md).then(done,function(){fb(md);});}else{fb(md);}
|
|
392
|
+
function fb(t){var x=document.createElement('textarea');x.value=t;document.body.appendChild(x);x.select();
|
|
393
|
+
try{document.execCommand('copy');}catch(e){}document.body.removeChild(x);done();}
|
|
394
|
+
});
|
|
395
|
+
})();
|
|
396
|
+
</script>
|
|
397
|
+
</body></html>
|