agent_eval_planner 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +6 -0
- data/.rspec +2 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +7 -0
- data/Gemfile +10 -0
- data/README.md +75 -9
- data/Rakefile +8 -0
- data/agent_eval_planner.gemspec +42 -0
- data/docs/.nojekyll +0 -0
- data/docs/examples/sample-contract.md +15 -0
- data/docs/examples/sample-plan.md +238 -0
- data/docs/examples/sample-remediations.md +245 -0
- data/docs/examples/sample-scope-01-native.json +37 -0
- data/docs/examples/sample-scope-01.json +32 -0
- data/docs/examples/sample-suite.jsonl +13 -0
- data/docs/favicon.svg +5 -0
- data/docs/index.html +469 -0
- data/docs/script.js +71 -0
- data/docs/styles.css +546 -0
- data/lib/agent_eval_planner/cli.rb +37 -8
- data/lib/agent_eval_planner/plan_renderer.rb +26 -5
- data/lib/agent_eval_planner/remediation_renderer.rb +12 -1
- data/lib/agent_eval_planner/suite_renderer.rb +49 -15
- data/lib/agent_eval_planner/suite_validator.rb +297 -62
- data/lib/agent_eval_planner/version.rb +1 -1
- data/lib/agent_eval_planner.rb +7 -2
- metadata +18 -1
data/docs/styles.css
ADDED
|
@@ -0,0 +1,546 @@
|
|
|
1
|
+
/* Agent Eval Planner — distinct from sibling gem sites */
|
|
2
|
+
:root {
|
|
3
|
+
--bg: #060807;
|
|
4
|
+
--bg-elevated: #0b100e;
|
|
5
|
+
--panel: #0e1512;
|
|
6
|
+
--panel-2: #121c18;
|
|
7
|
+
--line: rgba(61, 255, 154, 0.14);
|
|
8
|
+
--line-strong: rgba(61, 255, 154, 0.28);
|
|
9
|
+
--text: #e8f5ee;
|
|
10
|
+
--muted: #8aa396;
|
|
11
|
+
--dim: #5c7368;
|
|
12
|
+
--phosphor: #3dff9a;
|
|
13
|
+
--phosphor-dim: rgba(61, 255, 154, 0.12);
|
|
14
|
+
--amber: #ffb020;
|
|
15
|
+
--critical: #ff4d6d;
|
|
16
|
+
--cool: #7dd3fc;
|
|
17
|
+
--font: "Space Grotesk", system-ui, sans-serif;
|
|
18
|
+
--mono: "IBM Plex Mono", ui-monospace, monospace;
|
|
19
|
+
--header-h: 68px;
|
|
20
|
+
--max: 1120px;
|
|
21
|
+
--ease: cubic-bezier(0.22, 1, 0.36, 1);
|
|
22
|
+
--radius: 4px;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
|
26
|
+
html { scroll-behavior: smooth; scroll-padding-top: calc(var(--header-h) + 12px); }
|
|
27
|
+
body {
|
|
28
|
+
font-family: var(--font);
|
|
29
|
+
background: var(--bg);
|
|
30
|
+
color: var(--text);
|
|
31
|
+
line-height: 1.55;
|
|
32
|
+
-webkit-font-smoothing: antialiased;
|
|
33
|
+
overflow-x: hidden;
|
|
34
|
+
}
|
|
35
|
+
a { color: inherit; text-decoration: none; }
|
|
36
|
+
ul, ol { list-style: none; }
|
|
37
|
+
code, pre { font-family: var(--mono); }
|
|
38
|
+
|
|
39
|
+
.wrap {
|
|
40
|
+
width: min(100% - 2rem, var(--max));
|
|
41
|
+
margin-inline: auto;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/* Atmosphere */
|
|
45
|
+
.grid-bg {
|
|
46
|
+
position: fixed;
|
|
47
|
+
inset: 0;
|
|
48
|
+
z-index: 0;
|
|
49
|
+
pointer-events: none;
|
|
50
|
+
background-image:
|
|
51
|
+
linear-gradient(rgba(61, 255, 154, 0.04) 1px, transparent 1px),
|
|
52
|
+
linear-gradient(90deg, rgba(61, 255, 154, 0.04) 1px, transparent 1px);
|
|
53
|
+
background-size: 48px 48px;
|
|
54
|
+
mask-image: radial-gradient(ellipse at 50% 20%, black 20%, transparent 75%);
|
|
55
|
+
}
|
|
56
|
+
.scanlines {
|
|
57
|
+
position: fixed;
|
|
58
|
+
inset: 0;
|
|
59
|
+
z-index: 1;
|
|
60
|
+
pointer-events: none;
|
|
61
|
+
opacity: 0.04;
|
|
62
|
+
background: repeating-linear-gradient(
|
|
63
|
+
to bottom,
|
|
64
|
+
transparent 0,
|
|
65
|
+
transparent 2px,
|
|
66
|
+
rgba(0, 0, 0, 0.5) 3px
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
.vignette {
|
|
70
|
+
position: fixed;
|
|
71
|
+
inset: 0;
|
|
72
|
+
z-index: 1;
|
|
73
|
+
pointer-events: none;
|
|
74
|
+
background: radial-gradient(ellipse at center, transparent 40%, rgba(0, 0, 0, 0.55) 100%);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
.site-header,
|
|
78
|
+
main,
|
|
79
|
+
.site-footer { position: relative; z-index: 2; }
|
|
80
|
+
|
|
81
|
+
/* Header */
|
|
82
|
+
.site-header {
|
|
83
|
+
position: sticky;
|
|
84
|
+
top: 0;
|
|
85
|
+
height: var(--header-h);
|
|
86
|
+
border-bottom: 1px solid var(--line);
|
|
87
|
+
background: rgba(6, 8, 7, 0.85);
|
|
88
|
+
backdrop-filter: blur(12px);
|
|
89
|
+
}
|
|
90
|
+
.nav {
|
|
91
|
+
height: 100%;
|
|
92
|
+
display: flex;
|
|
93
|
+
align-items: center;
|
|
94
|
+
justify-content: space-between;
|
|
95
|
+
gap: 1rem;
|
|
96
|
+
}
|
|
97
|
+
.brand {
|
|
98
|
+
display: flex;
|
|
99
|
+
align-items: center;
|
|
100
|
+
gap: 0.65rem;
|
|
101
|
+
font-weight: 600;
|
|
102
|
+
letter-spacing: -0.02em;
|
|
103
|
+
}
|
|
104
|
+
.brand__mark {
|
|
105
|
+
width: 28px;
|
|
106
|
+
height: 28px;
|
|
107
|
+
border: 1px solid var(--phosphor);
|
|
108
|
+
color: var(--phosphor);
|
|
109
|
+
display: grid;
|
|
110
|
+
place-items: center;
|
|
111
|
+
font-family: var(--mono);
|
|
112
|
+
font-size: 0.7rem;
|
|
113
|
+
box-shadow: 0 0 16px rgba(61, 255, 154, 0.2);
|
|
114
|
+
}
|
|
115
|
+
.nav__links {
|
|
116
|
+
display: flex;
|
|
117
|
+
align-items: center;
|
|
118
|
+
gap: 1.25rem;
|
|
119
|
+
font-size: 0.9rem;
|
|
120
|
+
color: var(--muted);
|
|
121
|
+
}
|
|
122
|
+
.nav__links a:hover { color: var(--phosphor); }
|
|
123
|
+
.nav__cta {
|
|
124
|
+
border: 1px solid var(--line-strong);
|
|
125
|
+
padding: 0.4rem 0.75rem;
|
|
126
|
+
color: var(--phosphor) !important;
|
|
127
|
+
background: var(--phosphor-dim);
|
|
128
|
+
}
|
|
129
|
+
.nav__cta:hover { border-color: var(--phosphor); }
|
|
130
|
+
.nav__toggle {
|
|
131
|
+
display: none;
|
|
132
|
+
background: none;
|
|
133
|
+
border: 1px solid var(--line);
|
|
134
|
+
width: 42px;
|
|
135
|
+
height: 42px;
|
|
136
|
+
place-items: center;
|
|
137
|
+
gap: 5px;
|
|
138
|
+
flex-direction: column;
|
|
139
|
+
cursor: pointer;
|
|
140
|
+
}
|
|
141
|
+
.nav__toggle span {
|
|
142
|
+
display: block;
|
|
143
|
+
width: 18px;
|
|
144
|
+
height: 1px;
|
|
145
|
+
background: var(--text);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/* Hero */
|
|
149
|
+
.hero {
|
|
150
|
+
padding: 4.5rem 0 3.5rem;
|
|
151
|
+
display: grid;
|
|
152
|
+
grid-template-columns: 1.05fr 0.95fr;
|
|
153
|
+
gap: 2.5rem;
|
|
154
|
+
align-items: center;
|
|
155
|
+
}
|
|
156
|
+
.kicker {
|
|
157
|
+
display: inline-flex;
|
|
158
|
+
align-items: center;
|
|
159
|
+
gap: 0.5rem;
|
|
160
|
+
font-family: var(--mono);
|
|
161
|
+
font-size: 0.75rem;
|
|
162
|
+
letter-spacing: 0.08em;
|
|
163
|
+
text-transform: uppercase;
|
|
164
|
+
color: var(--phosphor);
|
|
165
|
+
margin-bottom: 1rem;
|
|
166
|
+
}
|
|
167
|
+
.kicker::before {
|
|
168
|
+
content: "";
|
|
169
|
+
width: 8px;
|
|
170
|
+
height: 8px;
|
|
171
|
+
background: var(--phosphor);
|
|
172
|
+
box-shadow: 0 0 12px var(--phosphor);
|
|
173
|
+
animation: pulse 1.8s ease-in-out infinite;
|
|
174
|
+
}
|
|
175
|
+
@keyframes pulse {
|
|
176
|
+
0%, 100% { opacity: 1; }
|
|
177
|
+
50% { opacity: 0.35; }
|
|
178
|
+
}
|
|
179
|
+
.hero h1 {
|
|
180
|
+
font-size: clamp(2.4rem, 5.2vw, 3.8rem);
|
|
181
|
+
line-height: 1.02;
|
|
182
|
+
letter-spacing: -0.04em;
|
|
183
|
+
font-weight: 600;
|
|
184
|
+
margin-bottom: 1rem;
|
|
185
|
+
}
|
|
186
|
+
.hero h1 em {
|
|
187
|
+
font-style: normal;
|
|
188
|
+
color: var(--phosphor);
|
|
189
|
+
text-shadow: 0 0 28px rgba(61, 255, 154, 0.25);
|
|
190
|
+
}
|
|
191
|
+
.hero__lead {
|
|
192
|
+
color: var(--muted);
|
|
193
|
+
font-size: 1.05rem;
|
|
194
|
+
max-width: 36rem;
|
|
195
|
+
margin-bottom: 1.5rem;
|
|
196
|
+
}
|
|
197
|
+
.hero__actions {
|
|
198
|
+
display: flex;
|
|
199
|
+
flex-wrap: wrap;
|
|
200
|
+
gap: 0.75rem;
|
|
201
|
+
margin-bottom: 1.75rem;
|
|
202
|
+
}
|
|
203
|
+
.btn {
|
|
204
|
+
display: inline-flex;
|
|
205
|
+
align-items: center;
|
|
206
|
+
justify-content: center;
|
|
207
|
+
padding: 0.7rem 1rem;
|
|
208
|
+
font-weight: 500;
|
|
209
|
+
border: 1px solid transparent;
|
|
210
|
+
transition: 0.2s var(--ease);
|
|
211
|
+
}
|
|
212
|
+
.btn--primary {
|
|
213
|
+
background: var(--phosphor);
|
|
214
|
+
color: #04110a;
|
|
215
|
+
}
|
|
216
|
+
.btn--primary:hover { filter: brightness(1.08); box-shadow: 0 0 24px rgba(61, 255, 154, 0.35); }
|
|
217
|
+
.btn--ghost {
|
|
218
|
+
border-color: var(--line-strong);
|
|
219
|
+
color: var(--text);
|
|
220
|
+
background: transparent;
|
|
221
|
+
}
|
|
222
|
+
.btn--ghost:hover { border-color: var(--phosphor); color: var(--phosphor); }
|
|
223
|
+
|
|
224
|
+
.meta-row {
|
|
225
|
+
display: flex;
|
|
226
|
+
flex-wrap: wrap;
|
|
227
|
+
gap: 0.5rem;
|
|
228
|
+
}
|
|
229
|
+
.chip {
|
|
230
|
+
font-family: var(--mono);
|
|
231
|
+
font-size: 0.72rem;
|
|
232
|
+
letter-spacing: 0.04em;
|
|
233
|
+
text-transform: uppercase;
|
|
234
|
+
border: 1px solid var(--line);
|
|
235
|
+
padding: 0.35rem 0.55rem;
|
|
236
|
+
color: var(--muted);
|
|
237
|
+
background: var(--bg-elevated);
|
|
238
|
+
}
|
|
239
|
+
.chip--hot { color: var(--critical); border-color: rgba(255, 77, 109, 0.35); }
|
|
240
|
+
.chip--ok { color: var(--phosphor); border-color: var(--line-strong); }
|
|
241
|
+
|
|
242
|
+
/* Terminal panel */
|
|
243
|
+
.panel {
|
|
244
|
+
background: linear-gradient(180deg, var(--panel-2), var(--panel));
|
|
245
|
+
border: 1px solid var(--line);
|
|
246
|
+
box-shadow:
|
|
247
|
+
0 0 0 1px rgba(0, 0, 0, 0.4),
|
|
248
|
+
0 24px 60px rgba(0, 0, 0, 0.45),
|
|
249
|
+
inset 0 1px 0 rgba(61, 255, 154, 0.08);
|
|
250
|
+
}
|
|
251
|
+
.panel__bar {
|
|
252
|
+
display: flex;
|
|
253
|
+
align-items: center;
|
|
254
|
+
gap: 0.5rem;
|
|
255
|
+
padding: 0.65rem 0.85rem;
|
|
256
|
+
border-bottom: 1px solid var(--line);
|
|
257
|
+
font-family: var(--mono);
|
|
258
|
+
font-size: 0.72rem;
|
|
259
|
+
color: var(--dim);
|
|
260
|
+
}
|
|
261
|
+
.panel__dot {
|
|
262
|
+
width: 7px;
|
|
263
|
+
height: 7px;
|
|
264
|
+
border-radius: 50%;
|
|
265
|
+
background: var(--dim);
|
|
266
|
+
}
|
|
267
|
+
.panel__dot:nth-child(1) { background: var(--critical); }
|
|
268
|
+
.panel__dot:nth-child(2) { background: var(--amber); }
|
|
269
|
+
.panel__dot:nth-child(3) { background: var(--phosphor); }
|
|
270
|
+
.panel__body {
|
|
271
|
+
padding: 1rem 1rem 1.15rem;
|
|
272
|
+
font-size: 0.78rem;
|
|
273
|
+
line-height: 1.55;
|
|
274
|
+
overflow-x: auto;
|
|
275
|
+
color: var(--muted);
|
|
276
|
+
}
|
|
277
|
+
.panel__body .ok { color: var(--phosphor); }
|
|
278
|
+
.panel__body .warn { color: var(--amber); }
|
|
279
|
+
.panel__body .bad { color: var(--critical); }
|
|
280
|
+
.panel__body .key { color: var(--cool); }
|
|
281
|
+
.panel__body .muted { color: var(--dim); }
|
|
282
|
+
|
|
283
|
+
.panel__link {
|
|
284
|
+
margin-left: auto;
|
|
285
|
+
color: var(--phosphor);
|
|
286
|
+
font-size: 0.7rem;
|
|
287
|
+
}
|
|
288
|
+
.panel__link:hover { text-decoration: underline; }
|
|
289
|
+
|
|
290
|
+
/* Evidence */
|
|
291
|
+
.evidence { display: grid; gap: 1rem; }
|
|
292
|
+
.tabs {
|
|
293
|
+
display: flex;
|
|
294
|
+
flex-wrap: wrap;
|
|
295
|
+
gap: 0.4rem;
|
|
296
|
+
}
|
|
297
|
+
.tab {
|
|
298
|
+
font-family: var(--mono);
|
|
299
|
+
font-size: 0.72rem;
|
|
300
|
+
letter-spacing: 0.03em;
|
|
301
|
+
padding: 0.45rem 0.7rem;
|
|
302
|
+
border: 1px solid var(--line);
|
|
303
|
+
background: var(--bg-elevated);
|
|
304
|
+
color: var(--muted);
|
|
305
|
+
cursor: pointer;
|
|
306
|
+
}
|
|
307
|
+
.tab:hover { color: var(--text); border-color: var(--line-strong); }
|
|
308
|
+
.tab.is-active {
|
|
309
|
+
color: #04110a;
|
|
310
|
+
background: var(--phosphor);
|
|
311
|
+
border-color: var(--phosphor);
|
|
312
|
+
}
|
|
313
|
+
.tab-panel[hidden] { display: none !important; }
|
|
314
|
+
.evidence__links {
|
|
315
|
+
display: flex;
|
|
316
|
+
flex-wrap: wrap;
|
|
317
|
+
gap: 0.75rem 1.25rem;
|
|
318
|
+
font-family: var(--mono);
|
|
319
|
+
font-size: 0.78rem;
|
|
320
|
+
}
|
|
321
|
+
.evidence__links a { color: var(--phosphor); border-bottom: 1px solid rgba(61, 255, 154, 0.35); }
|
|
322
|
+
.evidence__links a:hover { border-bottom-color: var(--phosphor); }
|
|
323
|
+
|
|
324
|
+
/* Sections */
|
|
325
|
+
.section { padding: 4rem 0; }
|
|
326
|
+
.section--band {
|
|
327
|
+
background: linear-gradient(180deg, rgba(14, 21, 18, 0.7), rgba(6, 8, 7, 0.2));
|
|
328
|
+
border-block: 1px solid var(--line);
|
|
329
|
+
}
|
|
330
|
+
.section__head {
|
|
331
|
+
margin-bottom: 2rem;
|
|
332
|
+
max-width: 40rem;
|
|
333
|
+
}
|
|
334
|
+
.section__head .kicker { margin-bottom: 0.6rem; }
|
|
335
|
+
.section__head h2 {
|
|
336
|
+
font-size: clamp(1.7rem, 3vw, 2.35rem);
|
|
337
|
+
letter-spacing: -0.03em;
|
|
338
|
+
margin-bottom: 0.65rem;
|
|
339
|
+
}
|
|
340
|
+
.section__head p { color: var(--muted); }
|
|
341
|
+
|
|
342
|
+
.prose {
|
|
343
|
+
color: var(--muted);
|
|
344
|
+
max-width: 46rem;
|
|
345
|
+
display: grid;
|
|
346
|
+
gap: 0.9rem;
|
|
347
|
+
}
|
|
348
|
+
.prose strong { color: var(--text); font-weight: 500; }
|
|
349
|
+
|
|
350
|
+
/* Pipeline */
|
|
351
|
+
.pipeline {
|
|
352
|
+
display: grid;
|
|
353
|
+
grid-template-columns: repeat(3, 1fr);
|
|
354
|
+
gap: 1rem;
|
|
355
|
+
}
|
|
356
|
+
.pipe {
|
|
357
|
+
border: 1px solid var(--line);
|
|
358
|
+
background: var(--bg-elevated);
|
|
359
|
+
padding: 1.25rem;
|
|
360
|
+
min-height: 100%;
|
|
361
|
+
transition: border-color 0.2s var(--ease), transform 0.2s var(--ease);
|
|
362
|
+
}
|
|
363
|
+
.pipe:hover {
|
|
364
|
+
border-color: var(--line-strong);
|
|
365
|
+
transform: translateY(-2px);
|
|
366
|
+
}
|
|
367
|
+
.pipe--focus {
|
|
368
|
+
border-color: rgba(61, 255, 154, 0.4);
|
|
369
|
+
background: linear-gradient(180deg, rgba(61, 255, 154, 0.08), var(--bg-elevated));
|
|
370
|
+
}
|
|
371
|
+
.pipe__id {
|
|
372
|
+
font-family: var(--mono);
|
|
373
|
+
font-size: 0.72rem;
|
|
374
|
+
color: var(--phosphor);
|
|
375
|
+
margin-bottom: 0.75rem;
|
|
376
|
+
}
|
|
377
|
+
.pipe h3 {
|
|
378
|
+
font-size: 1.05rem;
|
|
379
|
+
margin-bottom: 0.5rem;
|
|
380
|
+
letter-spacing: -0.02em;
|
|
381
|
+
}
|
|
382
|
+
.pipe p { color: var(--muted); font-size: 0.92rem; margin-bottom: 0.9rem; }
|
|
383
|
+
.pipe ul {
|
|
384
|
+
display: grid;
|
|
385
|
+
gap: 0.4rem;
|
|
386
|
+
color: var(--muted);
|
|
387
|
+
font-size: 0.88rem;
|
|
388
|
+
}
|
|
389
|
+
.pipe li::before {
|
|
390
|
+
content: "› ";
|
|
391
|
+
color: var(--phosphor);
|
|
392
|
+
font-family: var(--mono);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/* Taxonomy matrix */
|
|
396
|
+
.matrix {
|
|
397
|
+
display: grid;
|
|
398
|
+
grid-template-columns: repeat(4, 1fr);
|
|
399
|
+
gap: 0.75rem;
|
|
400
|
+
}
|
|
401
|
+
.cell {
|
|
402
|
+
border: 1px solid var(--line);
|
|
403
|
+
padding: 1rem;
|
|
404
|
+
background: rgba(11, 16, 14, 0.8);
|
|
405
|
+
}
|
|
406
|
+
.cell__code {
|
|
407
|
+
font-family: var(--mono);
|
|
408
|
+
font-size: 0.75rem;
|
|
409
|
+
color: var(--phosphor);
|
|
410
|
+
margin-bottom: 0.4rem;
|
|
411
|
+
}
|
|
412
|
+
.cell h3 { font-size: 0.98rem; margin-bottom: 0.35rem; }
|
|
413
|
+
.cell p { color: var(--muted); font-size: 0.85rem; }
|
|
414
|
+
.sev {
|
|
415
|
+
display: inline-block;
|
|
416
|
+
margin-top: 0.7rem;
|
|
417
|
+
font-family: var(--mono);
|
|
418
|
+
font-size: 0.68rem;
|
|
419
|
+
letter-spacing: 0.06em;
|
|
420
|
+
padding: 0.2rem 0.4rem;
|
|
421
|
+
border: 1px solid currentColor;
|
|
422
|
+
}
|
|
423
|
+
.sev--p0 { color: var(--critical); }
|
|
424
|
+
.sev--p1 { color: var(--amber); }
|
|
425
|
+
|
|
426
|
+
/* Checklist */
|
|
427
|
+
.check {
|
|
428
|
+
display: grid;
|
|
429
|
+
gap: 0.75rem;
|
|
430
|
+
}
|
|
431
|
+
.check li {
|
|
432
|
+
display: grid;
|
|
433
|
+
grid-template-columns: auto 1fr;
|
|
434
|
+
gap: 0.85rem;
|
|
435
|
+
border: 1px solid var(--line);
|
|
436
|
+
padding: 1rem 1.1rem;
|
|
437
|
+
background: var(--bg-elevated);
|
|
438
|
+
}
|
|
439
|
+
.check__mark {
|
|
440
|
+
font-family: var(--mono);
|
|
441
|
+
color: var(--phosphor);
|
|
442
|
+
margin-top: 0.15rem;
|
|
443
|
+
}
|
|
444
|
+
.check strong { display: block; margin-bottom: 0.2rem; }
|
|
445
|
+
.check span { color: var(--muted); font-size: 0.92rem; }
|
|
446
|
+
|
|
447
|
+
/* Install */
|
|
448
|
+
.install {
|
|
449
|
+
display: grid;
|
|
450
|
+
grid-template-columns: 0.9fr 1.1fr;
|
|
451
|
+
gap: 1.5rem;
|
|
452
|
+
align-items: start;
|
|
453
|
+
}
|
|
454
|
+
.cmd {
|
|
455
|
+
border: 1px solid var(--line);
|
|
456
|
+
background: #070b09;
|
|
457
|
+
margin-bottom: 0.75rem;
|
|
458
|
+
}
|
|
459
|
+
.cmd__head {
|
|
460
|
+
display: flex;
|
|
461
|
+
justify-content: space-between;
|
|
462
|
+
align-items: center;
|
|
463
|
+
padding: 0.55rem 0.8rem;
|
|
464
|
+
border-bottom: 1px solid var(--line);
|
|
465
|
+
font-family: var(--mono);
|
|
466
|
+
font-size: 0.72rem;
|
|
467
|
+
color: var(--dim);
|
|
468
|
+
}
|
|
469
|
+
.copy-btn {
|
|
470
|
+
background: transparent;
|
|
471
|
+
border: 1px solid var(--line);
|
|
472
|
+
color: var(--muted);
|
|
473
|
+
padding: 0.2rem 0.45rem;
|
|
474
|
+
cursor: pointer;
|
|
475
|
+
font-family: var(--mono);
|
|
476
|
+
font-size: 0.68rem;
|
|
477
|
+
}
|
|
478
|
+
.copy-btn:hover, .copy-btn.copied {
|
|
479
|
+
color: var(--phosphor);
|
|
480
|
+
border-color: var(--line-strong);
|
|
481
|
+
}
|
|
482
|
+
.cmd code {
|
|
483
|
+
display: block;
|
|
484
|
+
padding: 0.85rem 0.9rem;
|
|
485
|
+
color: var(--text);
|
|
486
|
+
font-size: 0.82rem;
|
|
487
|
+
overflow-x: auto;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
/* CTA */
|
|
491
|
+
.cta {
|
|
492
|
+
text-align: center;
|
|
493
|
+
padding: 3rem 1.25rem;
|
|
494
|
+
border: 1px solid var(--line-strong);
|
|
495
|
+
background:
|
|
496
|
+
linear-gradient(180deg, rgba(61, 255, 154, 0.06), transparent),
|
|
497
|
+
var(--bg-elevated);
|
|
498
|
+
}
|
|
499
|
+
.cta h2 { margin-bottom: 0.5rem; letter-spacing: -0.03em; }
|
|
500
|
+
.cta p { color: var(--muted); margin-bottom: 1.25rem; }
|
|
501
|
+
.cta__links { display: flex; justify-content: center; gap: 0.75rem; flex-wrap: wrap; }
|
|
502
|
+
|
|
503
|
+
.site-footer {
|
|
504
|
+
border-top: 1px solid var(--line);
|
|
505
|
+
padding: 1.5rem 0 2rem;
|
|
506
|
+
color: var(--dim);
|
|
507
|
+
font-size: 0.88rem;
|
|
508
|
+
}
|
|
509
|
+
.site-footer .wrap {
|
|
510
|
+
display: flex;
|
|
511
|
+
justify-content: space-between;
|
|
512
|
+
gap: 1rem;
|
|
513
|
+
flex-wrap: wrap;
|
|
514
|
+
}
|
|
515
|
+
.site-footer a:hover { color: var(--phosphor); }
|
|
516
|
+
|
|
517
|
+
/* Reveal */
|
|
518
|
+
.reveal {
|
|
519
|
+
opacity: 0;
|
|
520
|
+
transform: translateY(14px);
|
|
521
|
+
transition: opacity 0.7s var(--ease), transform 0.7s var(--ease);
|
|
522
|
+
}
|
|
523
|
+
.reveal.visible { opacity: 1; transform: none; }
|
|
524
|
+
.reveal--delay { transition-delay: 0.1s; }
|
|
525
|
+
.reveal--delay-2 { transition-delay: 0.18s; }
|
|
526
|
+
|
|
527
|
+
@media (max-width: 900px) {
|
|
528
|
+
.hero, .install, .pipeline { grid-template-columns: 1fr; }
|
|
529
|
+
.matrix { grid-template-columns: 1fr 1fr; }
|
|
530
|
+
.nav__toggle { display: grid; }
|
|
531
|
+
.nav__links {
|
|
532
|
+
position: fixed;
|
|
533
|
+
inset: var(--header-h) 0 auto 0;
|
|
534
|
+
background: rgba(6, 8, 7, 0.96);
|
|
535
|
+
border-bottom: 1px solid var(--line);
|
|
536
|
+
flex-direction: column;
|
|
537
|
+
align-items: stretch;
|
|
538
|
+
padding: 1rem;
|
|
539
|
+
display: none;
|
|
540
|
+
}
|
|
541
|
+
.nav__links.open { display: flex; }
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
@media (max-width: 560px) {
|
|
545
|
+
.matrix { grid-template-columns: 1fr; }
|
|
546
|
+
}
|
|
@@ -69,7 +69,7 @@ module AgentEvalPlanner
|
|
|
69
69
|
opts.on("--tools LIST", "Tools conhecidas (vírgula)") { |v| options[:tools] = v }
|
|
70
70
|
opts.on("--scope TEXT", "Escopo declarado (resumo)") { |v| options[:declared_scope] = v }
|
|
71
71
|
opts.on("--out-of-scope TEXT", "Fora de escopo declarado") { |v| options[:out_of_scope] = v }
|
|
72
|
-
opts.on("--harness NAME", "Harness alvo (generic|marketing-copilot)") { |v| options[:harness] = v }
|
|
72
|
+
opts.on("--harness NAME", "Harness alvo (generic|marketing-copilot|native)") { |v| options[:harness] = v }
|
|
73
73
|
opts.on("-o", "--output PATH", "Arquivo .md ou diretório de saída") { |v| options[:output] = v }
|
|
74
74
|
opts.on("--plan-only", "Gerar apenas o plano") { options[:plan_only] = true }
|
|
75
75
|
opts.on("--suite-only", "Gerar apenas a suite JSONL") { options[:suite_only] = true }
|
|
@@ -156,13 +156,28 @@ module AgentEvalPlanner
|
|
|
156
156
|
end
|
|
157
157
|
|
|
158
158
|
def run_validate(argv)
|
|
159
|
-
options = {
|
|
159
|
+
options = {
|
|
160
|
+
known_tools: [],
|
|
161
|
+
bound_tools: [],
|
|
162
|
+
agent_has_no_tools: false,
|
|
163
|
+
target_components: [],
|
|
164
|
+
suites: [],
|
|
165
|
+
schema: "auto"
|
|
166
|
+
}
|
|
160
167
|
parser = OptionParser.new do |opts|
|
|
161
168
|
opts.banner = "Usage: agent-eval-planner validate [options] <suite.jsonl>\n\n"
|
|
162
|
-
opts.on("--known-tools LIST", "
|
|
163
|
-
options[:known_tools] = v
|
|
169
|
+
opts.on("--known-tools LIST", "Catálogo do produto (vírgula)") do |v|
|
|
170
|
+
options[:known_tools] = split_csv(v)
|
|
171
|
+
end
|
|
172
|
+
opts.on("--bound-tools LIST", "Tools ligadas a este alvo (vírgula)") do |v|
|
|
173
|
+
options[:bound_tools] = split_csv(v)
|
|
164
174
|
end
|
|
165
175
|
opts.on("--agent-has-no-tools", "Agente sem tools") { options[:agent_has_no_tools] = true }
|
|
176
|
+
opts.on("--target-components LIST", "target_component ou target_agent aceitos") do |v|
|
|
177
|
+
options[:target_components] = split_csv(v)
|
|
178
|
+
end
|
|
179
|
+
opts.on("--suites LIST", "Valores nativos de suite") { |v| options[:suites] = split_csv(v) }
|
|
180
|
+
opts.on("--schema NAME", "auto, legacy ou native") { |v| options[:schema] = v }
|
|
166
181
|
opts.on("-h", "--help", "Ajuda") do
|
|
167
182
|
puts opts
|
|
168
183
|
exit
|
|
@@ -172,23 +187,37 @@ module AgentEvalPlanner
|
|
|
172
187
|
suite = argv.first
|
|
173
188
|
abort parser.help if suite.nil?
|
|
174
189
|
|
|
175
|
-
|
|
190
|
+
validator = SuiteValidator.new(
|
|
176
191
|
path: suite,
|
|
177
192
|
known_tools: options[:known_tools],
|
|
178
|
-
|
|
179
|
-
|
|
193
|
+
bound_tools: options[:bound_tools],
|
|
194
|
+
agent_has_no_tools: options[:agent_has_no_tools],
|
|
195
|
+
target_components: options[:target_components],
|
|
196
|
+
suites: options[:suites],
|
|
197
|
+
schema: options[:schema]
|
|
198
|
+
)
|
|
199
|
+
errors = validator.validate
|
|
180
200
|
|
|
201
|
+
validator.warnings.each { |warning| warn warning }
|
|
181
202
|
if errors.any?
|
|
182
203
|
warn "INVALID: #{suite} (#{errors.size} error(s))"
|
|
183
204
|
errors.each { |e| warn " - #{e}" }
|
|
184
205
|
exit 1
|
|
185
206
|
end
|
|
186
207
|
|
|
187
|
-
|
|
208
|
+
if validator.warnings.any?
|
|
209
|
+
puts "OK WITH WARNINGS: #{suite} (#{validator.warnings.size} warning(s))"
|
|
210
|
+
else
|
|
211
|
+
puts "OK: #{suite}"
|
|
212
|
+
end
|
|
188
213
|
rescue Error => e
|
|
189
214
|
warn "Erro: #{e.message}"
|
|
190
215
|
exit 1
|
|
191
216
|
end
|
|
217
|
+
|
|
218
|
+
def split_csv(value)
|
|
219
|
+
value.to_s.split(",").map(&:strip).reject(&:empty?)
|
|
220
|
+
end
|
|
192
221
|
end
|
|
193
222
|
end
|
|
194
223
|
|
|
@@ -29,8 +29,7 @@ module AgentEvalPlanner
|
|
|
29
29
|
|
|
30
30
|
def introduction
|
|
31
31
|
tools = analyzer.tools.empty? ? "_(nenhuma)_" : analyzer.tools.map { |t| "`#{t}`" }.join(", ")
|
|
32
|
-
|
|
33
|
-
<<~MD.strip
|
|
32
|
+
head = <<~MD.strip
|
|
34
33
|
## Introdução
|
|
35
34
|
|
|
36
35
|
#{analyzer.introduction}
|
|
@@ -40,13 +39,15 @@ module AgentEvalPlanner
|
|
|
40
39
|
**Contrato analisado:** `#{analyzer.source_path || "inline"}`
|
|
41
40
|
|
|
42
41
|
**Harness alvo:** #{analyzer.harness}
|
|
43
|
-
|
|
42
|
+
MD
|
|
43
|
+
tail = <<~MD.strip
|
|
44
44
|
**Escopo declarado (resumo):** #{analyzer.declared_scope}
|
|
45
45
|
|
|
46
46
|
**Fora de escopo declarado:** #{analyzer.out_of_scope}
|
|
47
47
|
|
|
48
48
|
**Tools / capabilities:** #{tools}
|
|
49
49
|
MD
|
|
50
|
+
[head, harness_note, tail].reject(&:empty?).join("\n\n")
|
|
50
51
|
end
|
|
51
52
|
|
|
52
53
|
def eval_sections
|
|
@@ -63,7 +64,7 @@ module AgentEvalPlanner
|
|
|
63
64
|
end
|
|
64
65
|
|
|
65
66
|
def render_vector(vector)
|
|
66
|
-
<<~MD.strip
|
|
67
|
+
body = <<~MD.strip
|
|
67
68
|
#### #{vector.id} — #{vector.name}
|
|
68
69
|
|
|
69
70
|
- **Severidade:** #{vector.severity}
|
|
@@ -74,8 +75,28 @@ module AgentEvalPlanner
|
|
|
74
75
|
```
|
|
75
76
|
- **Validação esperada:** #{vector.expected_validation}
|
|
76
77
|
- **Critério de falha:** #{vector.failure_criteria}
|
|
77
|
-
- **Observação:** #{vector.notes.to_s.empty? ? "—" : vector.notes}
|
|
78
78
|
MD
|
|
79
|
+
observation = "- **Observação:** #{vector.notes.to_s.empty? ? "—" : vector.notes}"
|
|
80
|
+
[body, vector_harness_lines(vector), observation].reject(&:empty?).join("\n")
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def harness_note
|
|
84
|
+
return "" unless analyzer.harness == "native"
|
|
85
|
+
|
|
86
|
+
<<~MD.strip
|
|
87
|
+
**Schema da suite:** nativo (`target_component`, `turns` com `sequence` desde 0, `suite=regression`, `metadata.vector_id`). `golden` só entra com curadoria.
|
|
88
|
+
|
|
89
|
+
**Camada:** marque `prompt`, `gate` ou `ambos` depois de ler o routing do produto. Sem gate estreito no specialist, a camada é `prompt`.
|
|
90
|
+
|
|
91
|
+
**O avaliador não pontua:** teto de chamadas, substring proibida na resposta e pedido misto além do par `refusal` / `expected_tools`.
|
|
92
|
+
MD
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def vector_harness_lines(vector)
|
|
96
|
+
return "" unless analyzer.harness == "native"
|
|
97
|
+
|
|
98
|
+
"- **IDs:** `#{vector.id}` no plano e `metadata.vector_id` na suíte\n" \
|
|
99
|
+
"- **Camada:** prompt (confirme no routing se o gate já cobre a frase)"
|
|
79
100
|
end
|
|
80
101
|
|
|
81
102
|
def summary_table
|
|
@@ -29,12 +29,23 @@ module AgentEvalPlanner
|
|
|
29
29
|
|
|
30
30
|
def executive_summary
|
|
31
31
|
p0 = vectors.count { |v| v.severity == "P0" }
|
|
32
|
-
<<~MD.strip
|
|
32
|
+
text = <<~MD.strip
|
|
33
33
|
## Resumo executivo
|
|
34
34
|
|
|
35
35
|
Plano gerado para **#{analyzer.agent_name}** (#{team}): #{vectors.size} vetores,
|
|
36
36
|
dos quais #{p0} P0. Snippets são referência — não PR pronto para merge.
|
|
37
37
|
MD
|
|
38
|
+
note = harness_summary
|
|
39
|
+
note.empty? ? text : "#{text}\n\n#{note}"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def harness_summary
|
|
43
|
+
return "" unless analyzer.harness == "native"
|
|
44
|
+
|
|
45
|
+
"Suite nativa: `target_component` + `turns` + `suite=regression`, " \
|
|
46
|
+
"ligada ao plano por `metadata.vector_id`. " \
|
|
47
|
+
"Se o texto de EXFIL mora num helper compartilhado, " \
|
|
48
|
+
"marque o raio: o mesmo trecho vale para os agentes que o importam."
|
|
38
49
|
end
|
|
39
50
|
|
|
40
51
|
def quick_wins
|