@pasko70/pibo 1.9.8 → 1.9.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +183 -183
- package/context/codex-base-prompt.md +148 -148
- package/context/compute-worker.md +23 -23
- package/context/pibo-compaction-prompt.md +100 -100
- package/context/pibo-native-tooling.md +18 -18
- package/context/pibo-system-prompt.md +77 -77
- package/dist/apps/chat/agent-store.js +248 -248
- package/dist/apps/chat/data/project-service.js +168 -168
- package/dist/apps/chat/data/read-state-service.js +18 -18
- package/dist/apps/chat/data/session-query-service.js +25 -25
- package/dist/apps/chat/data/timeline-query-service.js +14 -14
- package/dist/apps/chat/static-assets.js +854 -854
- package/dist/apps/chat/workflow-persistence.js +255 -255
- package/dist/apps/chat-ui/index.html +16 -16
- package/dist/apps/chat-ui/manifest.webmanifest +25 -25
- package/dist/apps/chat-ui/sw.js +47 -47
- package/dist/apps/chat-vscode-web/index.html +12 -12
- package/dist/apps/cli-ui/cliSessionsCommand.js +23 -23
- package/dist/apps/context-files-ui/index.html +11 -11
- package/dist/cli.js +39 -39
- package/dist/compute/cli.js +54 -54
- package/dist/cron/cli.js +15 -15
- package/dist/cron/store.js +49 -49
- package/dist/data/cli.js +23 -23
- package/dist/data/event-log.js +23 -23
- package/dist/data/message-store.js +20 -20
- package/dist/data/navigation-store.js +9 -9
- package/dist/data/observation-store.js +4 -4
- package/dist/data/payload-store.js +17 -17
- package/dist/data/schema.js +431 -431
- package/dist/data/session-store.js +4 -4
- package/dist/data/telemetry-queries.js +54 -54
- package/dist/data/telemetry.js +197 -197
- package/dist/debug/events.js +12 -12
- package/dist/debug/failures.js +6 -6
- package/dist/debug/index.js +227 -227
- package/dist/debug/messages.js +6 -6
- package/dist/debug/pty.js +124 -124
- package/dist/debug/session.js +29 -29
- package/dist/debug/tools.js +5 -5
- package/dist/debug/web-snapshot-browser-scripts.js +294 -294
- package/dist/debug/web-streaming-browser-library.js +925 -925
- package/dist/debug/web-streaming-browser-scripts.js +232 -232
- package/dist/debug/web-streaming-provider-telemetry.js +4 -4
- package/dist/debug/web.js +93 -93
- package/dist/gateway/cli.js +19 -19
- package/dist/mcp/config-command.js +53 -53
- package/dist/mcp/index.js +21 -21
- package/dist/mcp/registry.js +11 -11
- package/dist/pi-packages/cli.js +11 -11
- package/dist/plugins/context-files-store.js +110 -110
- package/dist/plugins/context-files.js +4 -4
- package/dist/ralph/cli.js +18 -18
- package/dist/ralph/templates.js +140 -140
- package/dist/reliability/store.js +226 -226
- package/dist/sessions/pibo-data-store.js +16 -16
- package/dist/sessions/sqlite-store.js +53 -53
- package/dist/setup/cli.js +58 -58
- package/dist/tools/agent-browser-wrapper.js +80 -80
- package/dist/tools/browser-use-cdp.js +12 -12
- package/dist/tools/browser-use-wrapper.js +762 -762
- package/dist/tools/guides.js +538 -538
- package/dist/tools/index.js +99 -99
- package/dist/tools/runtime/node-worker-source.js +205 -205
- package/dist/tools/runtime/python-worker-source.js +177 -177
- package/dist/vscode/cli.js +9 -9
- package/dist/web-annotations/cdp.js +900 -900
- package/dist/web-annotations/store.js +96 -96
- package/docs/README.md +23 -23
- package/docs/ops/install-developer-host.md +112 -112
- package/docs/ops/install-user-host.md +96 -96
- package/docs/ops/upgrade-user-to-developer-host.md +69 -69
- package/docs/ops/vscode-extension-release.md +160 -160
- package/package.json +95 -95
- package/skills/builtin/graphify/SKILL.md +52 -52
- package/skills/builtin/pi-agent-harness/SKILL.md +319 -319
- package/skills/builtin/pi-agent-harness/agents/openai.yaml +4 -4
- package/skills/builtin/pibo-docker-system/SKILL.md +170 -170
- package/skills/builtin/pibo-spec-writing/SKILL.md +330 -330
- package/skills/builtin/prd/SKILL.md +143 -143
- package/skills/builtin/ralph-loop/SKILL.md +359 -359
- package/skills/builtin/ralph-prd-json/SKILL.md +123 -123
- package/skills/builtin/skill-creator/LICENSE.txt +201 -201
- package/skills/builtin/skill-creator/SKILL.md +513 -513
- package/skills/builtin/skill-creator/agents/analyzer.md +274 -274
- package/skills/builtin/skill-creator/agents/comparator.md +202 -202
- package/skills/builtin/skill-creator/agents/grader.md +223 -223
- package/skills/builtin/skill-creator/assets/eval_review.html +146 -146
- package/skills/builtin/skill-creator/eval-viewer/generate_review.py +471 -471
- package/skills/builtin/skill-creator/eval-viewer/viewer.html +1325 -1325
- package/skills/builtin/skill-creator/references/schemas.md +430 -430
- package/skills/builtin/skill-creator/scripts/aggregate_benchmark.py +401 -401
- package/skills/builtin/skill-creator/scripts/generate_report.py +326 -326
- package/skills/builtin/skill-creator/scripts/improve_description.py +247 -247
- package/skills/builtin/skill-creator/scripts/package_skill.py +136 -136
- package/skills/builtin/skill-creator/scripts/quick_validate.py +102 -102
- package/skills/builtin/skill-creator/scripts/run_eval.py +310 -310
- package/skills/builtin/skill-creator/scripts/run_loop.py +328 -328
- package/skills/builtin/skill-creator/scripts/utils.py +47 -47
- package/skills/builtin/web-annotations/SKILL.md +93 -93
- package/src/mcp/LICENSE.mcp-cli +21 -21
|
@@ -1,326 +1,326 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Generate an HTML report from run_loop.py output.
|
|
3
|
-
|
|
4
|
-
Takes the JSON output from run_loop.py and generates a visual HTML report
|
|
5
|
-
showing each description attempt with check/x for each test case.
|
|
6
|
-
Distinguishes between train and test queries.
|
|
7
|
-
"""
|
|
8
|
-
|
|
9
|
-
import argparse
|
|
10
|
-
import html
|
|
11
|
-
import json
|
|
12
|
-
import sys
|
|
13
|
-
from pathlib import Path
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def generate_html(data: dict, auto_refresh: bool = False, skill_name: str = "") -> str:
|
|
17
|
-
"""Generate HTML report from loop output data. If auto_refresh is True, adds a meta refresh tag."""
|
|
18
|
-
history = data.get("history", [])
|
|
19
|
-
holdout = data.get("holdout", 0)
|
|
20
|
-
title_prefix = html.escape(skill_name + " \u2014 ") if skill_name else ""
|
|
21
|
-
|
|
22
|
-
# Get all unique queries from train and test sets, with should_trigger info
|
|
23
|
-
train_queries: list[dict] = []
|
|
24
|
-
test_queries: list[dict] = []
|
|
25
|
-
if history:
|
|
26
|
-
for r in history[0].get("train_results", history[0].get("results", [])):
|
|
27
|
-
train_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)})
|
|
28
|
-
if history[0].get("test_results"):
|
|
29
|
-
for r in history[0].get("test_results", []):
|
|
30
|
-
test_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)})
|
|
31
|
-
|
|
32
|
-
refresh_tag = ' <meta http-equiv="refresh" content="5">\n' if auto_refresh else ""
|
|
33
|
-
|
|
34
|
-
html_parts = ["""<!DOCTYPE html>
|
|
35
|
-
<html>
|
|
36
|
-
<head>
|
|
37
|
-
<meta charset="utf-8">
|
|
38
|
-
""" + refresh_tag + """ <title>""" + title_prefix + """Skill Description Optimization</title>
|
|
39
|
-
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
40
|
-
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
41
|
-
<link href="https://fonts.googleapis.com/css2?family=Poppins:wght@500;600&family=Lora:wght@400;500&display=swap" rel="stylesheet">
|
|
42
|
-
<style>
|
|
43
|
-
body {
|
|
44
|
-
font-family: 'Lora', Georgia, serif;
|
|
45
|
-
max-width: 100%;
|
|
46
|
-
margin: 0 auto;
|
|
47
|
-
padding: 20px;
|
|
48
|
-
background: #faf9f5;
|
|
49
|
-
color: #141413;
|
|
50
|
-
}
|
|
51
|
-
h1 { font-family: 'Poppins', sans-serif; color: #141413; }
|
|
52
|
-
.explainer {
|
|
53
|
-
background: white;
|
|
54
|
-
padding: 15px;
|
|
55
|
-
border-radius: 6px;
|
|
56
|
-
margin-bottom: 20px;
|
|
57
|
-
border: 1px solid #e8e6dc;
|
|
58
|
-
color: #b0aea5;
|
|
59
|
-
font-size: 0.875rem;
|
|
60
|
-
line-height: 1.6;
|
|
61
|
-
}
|
|
62
|
-
.summary {
|
|
63
|
-
background: white;
|
|
64
|
-
padding: 15px;
|
|
65
|
-
border-radius: 6px;
|
|
66
|
-
margin-bottom: 20px;
|
|
67
|
-
border: 1px solid #e8e6dc;
|
|
68
|
-
}
|
|
69
|
-
.summary p { margin: 5px 0; }
|
|
70
|
-
.best { color: #788c5d; font-weight: bold; }
|
|
71
|
-
.table-container {
|
|
72
|
-
overflow-x: auto;
|
|
73
|
-
width: 100%;
|
|
74
|
-
}
|
|
75
|
-
table {
|
|
76
|
-
border-collapse: collapse;
|
|
77
|
-
background: white;
|
|
78
|
-
border: 1px solid #e8e6dc;
|
|
79
|
-
border-radius: 6px;
|
|
80
|
-
font-size: 12px;
|
|
81
|
-
min-width: 100%;
|
|
82
|
-
}
|
|
83
|
-
th, td {
|
|
84
|
-
padding: 8px;
|
|
85
|
-
text-align: left;
|
|
86
|
-
border: 1px solid #e8e6dc;
|
|
87
|
-
white-space: normal;
|
|
88
|
-
word-wrap: break-word;
|
|
89
|
-
}
|
|
90
|
-
th {
|
|
91
|
-
font-family: 'Poppins', sans-serif;
|
|
92
|
-
background: #141413;
|
|
93
|
-
color: #faf9f5;
|
|
94
|
-
font-weight: 500;
|
|
95
|
-
}
|
|
96
|
-
th.test-col {
|
|
97
|
-
background: #6a9bcc;
|
|
98
|
-
}
|
|
99
|
-
th.query-col { min-width: 200px; }
|
|
100
|
-
td.description {
|
|
101
|
-
font-family: monospace;
|
|
102
|
-
font-size: 11px;
|
|
103
|
-
word-wrap: break-word;
|
|
104
|
-
max-width: 400px;
|
|
105
|
-
}
|
|
106
|
-
td.result {
|
|
107
|
-
text-align: center;
|
|
108
|
-
font-size: 16px;
|
|
109
|
-
min-width: 40px;
|
|
110
|
-
}
|
|
111
|
-
td.test-result {
|
|
112
|
-
background: #f0f6fc;
|
|
113
|
-
}
|
|
114
|
-
.pass { color: #788c5d; }
|
|
115
|
-
.fail { color: #c44; }
|
|
116
|
-
.rate {
|
|
117
|
-
font-size: 9px;
|
|
118
|
-
color: #b0aea5;
|
|
119
|
-
display: block;
|
|
120
|
-
}
|
|
121
|
-
tr:hover { background: #faf9f5; }
|
|
122
|
-
.score {
|
|
123
|
-
display: inline-block;
|
|
124
|
-
padding: 2px 6px;
|
|
125
|
-
border-radius: 4px;
|
|
126
|
-
font-weight: bold;
|
|
127
|
-
font-size: 11px;
|
|
128
|
-
}
|
|
129
|
-
.score-good { background: #eef2e8; color: #788c5d; }
|
|
130
|
-
.score-ok { background: #fef3c7; color: #d97706; }
|
|
131
|
-
.score-bad { background: #fceaea; color: #c44; }
|
|
132
|
-
.train-label { color: #b0aea5; font-size: 10px; }
|
|
133
|
-
.test-label { color: #6a9bcc; font-size: 10px; font-weight: bold; }
|
|
134
|
-
.best-row { background: #f5f8f2; }
|
|
135
|
-
th.positive-col { border-bottom: 3px solid #788c5d; }
|
|
136
|
-
th.negative-col { border-bottom: 3px solid #c44; }
|
|
137
|
-
th.test-col.positive-col { border-bottom: 3px solid #788c5d; }
|
|
138
|
-
th.test-col.negative-col { border-bottom: 3px solid #c44; }
|
|
139
|
-
.legend { font-family: 'Poppins', sans-serif; display: flex; gap: 20px; margin-bottom: 10px; font-size: 13px; align-items: center; }
|
|
140
|
-
.legend-item { display: flex; align-items: center; gap: 6px; }
|
|
141
|
-
.legend-swatch { width: 16px; height: 16px; border-radius: 3px; display: inline-block; }
|
|
142
|
-
.swatch-positive { background: #141413; border-bottom: 3px solid #788c5d; }
|
|
143
|
-
.swatch-negative { background: #141413; border-bottom: 3px solid #c44; }
|
|
144
|
-
.swatch-test { background: #6a9bcc; }
|
|
145
|
-
.swatch-train { background: #141413; }
|
|
146
|
-
</style>
|
|
147
|
-
</head>
|
|
148
|
-
<body>
|
|
149
|
-
<h1>""" + title_prefix + """Skill Description Optimization</h1>
|
|
150
|
-
<div class="explainer">
|
|
151
|
-
<strong>Optimizing your skill's description.</strong> This page updates automatically as Claude tests different versions of your skill's description. Each row is an iteration — a new description attempt. The columns show test queries: green checkmarks mean the skill triggered correctly (or correctly didn't trigger), red crosses mean it got it wrong. The "Train" score shows performance on queries used to improve the description; the "Test" score shows performance on held-out queries the optimizer hasn't seen. When it's done, Claude will apply the best-performing description to your skill.
|
|
152
|
-
</div>
|
|
153
|
-
"""]
|
|
154
|
-
|
|
155
|
-
# Summary section
|
|
156
|
-
best_test_score = data.get('best_test_score')
|
|
157
|
-
best_train_score = data.get('best_train_score')
|
|
158
|
-
html_parts.append(f"""
|
|
159
|
-
<div class="summary">
|
|
160
|
-
<p><strong>Original:</strong> {html.escape(data.get('original_description', 'N/A'))}</p>
|
|
161
|
-
<p class="best"><strong>Best:</strong> {html.escape(data.get('best_description', 'N/A'))}</p>
|
|
162
|
-
<p><strong>Best Score:</strong> {data.get('best_score', 'N/A')} {'(test)' if best_test_score else '(train)'}</p>
|
|
163
|
-
<p><strong>Iterations:</strong> {data.get('iterations_run', 0)} | <strong>Train:</strong> {data.get('train_size', '?')} | <strong>Test:</strong> {data.get('test_size', '?')}</p>
|
|
164
|
-
</div>
|
|
165
|
-
""")
|
|
166
|
-
|
|
167
|
-
# Legend
|
|
168
|
-
html_parts.append("""
|
|
169
|
-
<div class="legend">
|
|
170
|
-
<span style="font-weight:600">Query columns:</span>
|
|
171
|
-
<span class="legend-item"><span class="legend-swatch swatch-positive"></span> Should trigger</span>
|
|
172
|
-
<span class="legend-item"><span class="legend-swatch swatch-negative"></span> Should NOT trigger</span>
|
|
173
|
-
<span class="legend-item"><span class="legend-swatch swatch-train"></span> Train</span>
|
|
174
|
-
<span class="legend-item"><span class="legend-swatch swatch-test"></span> Test</span>
|
|
175
|
-
</div>
|
|
176
|
-
""")
|
|
177
|
-
|
|
178
|
-
# Table header
|
|
179
|
-
html_parts.append("""
|
|
180
|
-
<div class="table-container">
|
|
181
|
-
<table>
|
|
182
|
-
<thead>
|
|
183
|
-
<tr>
|
|
184
|
-
<th>Iter</th>
|
|
185
|
-
<th>Train</th>
|
|
186
|
-
<th>Test</th>
|
|
187
|
-
<th class="query-col">Description</th>
|
|
188
|
-
""")
|
|
189
|
-
|
|
190
|
-
# Add column headers for train queries
|
|
191
|
-
for qinfo in train_queries:
|
|
192
|
-
polarity = "positive-col" if qinfo["should_trigger"] else "negative-col"
|
|
193
|
-
html_parts.append(f' <th class="{polarity}">{html.escape(qinfo["query"])}</th>\n')
|
|
194
|
-
|
|
195
|
-
# Add column headers for test queries (different color)
|
|
196
|
-
for qinfo in test_queries:
|
|
197
|
-
polarity = "positive-col" if qinfo["should_trigger"] else "negative-col"
|
|
198
|
-
html_parts.append(f' <th class="test-col {polarity}">{html.escape(qinfo["query"])}</th>\n')
|
|
199
|
-
|
|
200
|
-
html_parts.append(""" </tr>
|
|
201
|
-
</thead>
|
|
202
|
-
<tbody>
|
|
203
|
-
""")
|
|
204
|
-
|
|
205
|
-
# Find best iteration for highlighting
|
|
206
|
-
if test_queries:
|
|
207
|
-
best_iter = max(history, key=lambda h: h.get("test_passed") or 0).get("iteration")
|
|
208
|
-
else:
|
|
209
|
-
best_iter = max(history, key=lambda h: h.get("train_passed", h.get("passed", 0))).get("iteration")
|
|
210
|
-
|
|
211
|
-
# Add rows for each iteration
|
|
212
|
-
for h in history:
|
|
213
|
-
iteration = h.get("iteration", "?")
|
|
214
|
-
train_passed = h.get("train_passed", h.get("passed", 0))
|
|
215
|
-
train_total = h.get("train_total", h.get("total", 0))
|
|
216
|
-
test_passed = h.get("test_passed")
|
|
217
|
-
test_total = h.get("test_total")
|
|
218
|
-
description = h.get("description", "")
|
|
219
|
-
train_results = h.get("train_results", h.get("results", []))
|
|
220
|
-
test_results = h.get("test_results", [])
|
|
221
|
-
|
|
222
|
-
# Create lookups for results by query
|
|
223
|
-
train_by_query = {r["query"]: r for r in train_results}
|
|
224
|
-
test_by_query = {r["query"]: r for r in test_results} if test_results else {}
|
|
225
|
-
|
|
226
|
-
# Compute aggregate correct/total runs across all retries
|
|
227
|
-
def aggregate_runs(results: list[dict]) -> tuple[int, int]:
|
|
228
|
-
correct = 0
|
|
229
|
-
total = 0
|
|
230
|
-
for r in results:
|
|
231
|
-
runs = r.get("runs", 0)
|
|
232
|
-
triggers = r.get("triggers", 0)
|
|
233
|
-
total += runs
|
|
234
|
-
if r.get("should_trigger", True):
|
|
235
|
-
correct += triggers
|
|
236
|
-
else:
|
|
237
|
-
correct += runs - triggers
|
|
238
|
-
return correct, total
|
|
239
|
-
|
|
240
|
-
train_correct, train_runs = aggregate_runs(train_results)
|
|
241
|
-
test_correct, test_runs = aggregate_runs(test_results)
|
|
242
|
-
|
|
243
|
-
# Determine score classes
|
|
244
|
-
def score_class(correct: int, total: int) -> str:
|
|
245
|
-
if total > 0:
|
|
246
|
-
ratio = correct / total
|
|
247
|
-
if ratio >= 0.8:
|
|
248
|
-
return "score-good"
|
|
249
|
-
elif ratio >= 0.5:
|
|
250
|
-
return "score-ok"
|
|
251
|
-
return "score-bad"
|
|
252
|
-
|
|
253
|
-
train_class = score_class(train_correct, train_runs)
|
|
254
|
-
test_class = score_class(test_correct, test_runs)
|
|
255
|
-
|
|
256
|
-
row_class = "best-row" if iteration == best_iter else ""
|
|
257
|
-
|
|
258
|
-
html_parts.append(f""" <tr class="{row_class}">
|
|
259
|
-
<td>{iteration}</td>
|
|
260
|
-
<td><span class="score {train_class}">{train_correct}/{train_runs}</span></td>
|
|
261
|
-
<td><span class="score {test_class}">{test_correct}/{test_runs}</span></td>
|
|
262
|
-
<td class="description">{html.escape(description)}</td>
|
|
263
|
-
""")
|
|
264
|
-
|
|
265
|
-
# Add result for each train query
|
|
266
|
-
for qinfo in train_queries:
|
|
267
|
-
r = train_by_query.get(qinfo["query"], {})
|
|
268
|
-
did_pass = r.get("pass", False)
|
|
269
|
-
triggers = r.get("triggers", 0)
|
|
270
|
-
runs = r.get("runs", 0)
|
|
271
|
-
|
|
272
|
-
icon = "✓" if did_pass else "✗"
|
|
273
|
-
css_class = "pass" if did_pass else "fail"
|
|
274
|
-
|
|
275
|
-
html_parts.append(f' <td class="result {css_class}">{icon}<span class="rate">{triggers}/{runs}</span></td>\n')
|
|
276
|
-
|
|
277
|
-
# Add result for each test query (with different background)
|
|
278
|
-
for qinfo in test_queries:
|
|
279
|
-
r = test_by_query.get(qinfo["query"], {})
|
|
280
|
-
did_pass = r.get("pass", False)
|
|
281
|
-
triggers = r.get("triggers", 0)
|
|
282
|
-
runs = r.get("runs", 0)
|
|
283
|
-
|
|
284
|
-
icon = "✓" if did_pass else "✗"
|
|
285
|
-
css_class = "pass" if did_pass else "fail"
|
|
286
|
-
|
|
287
|
-
html_parts.append(f' <td class="result test-result {css_class}">{icon}<span class="rate">{triggers}/{runs}</span></td>\n')
|
|
288
|
-
|
|
289
|
-
html_parts.append(" </tr>\n")
|
|
290
|
-
|
|
291
|
-
html_parts.append(""" </tbody>
|
|
292
|
-
</table>
|
|
293
|
-
</div>
|
|
294
|
-
""")
|
|
295
|
-
|
|
296
|
-
html_parts.append("""
|
|
297
|
-
</body>
|
|
298
|
-
</html>
|
|
299
|
-
""")
|
|
300
|
-
|
|
301
|
-
return "".join(html_parts)
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
def main():
|
|
305
|
-
parser = argparse.ArgumentParser(description="Generate HTML report from run_loop output")
|
|
306
|
-
parser.add_argument("input", help="Path to JSON output from run_loop.py (or - for stdin)")
|
|
307
|
-
parser.add_argument("-o", "--output", default=None, help="Output HTML file (default: stdout)")
|
|
308
|
-
parser.add_argument("--skill-name", default="", help="Skill name to include in the report title")
|
|
309
|
-
args = parser.parse_args()
|
|
310
|
-
|
|
311
|
-
if args.input == "-":
|
|
312
|
-
data = json.load(sys.stdin)
|
|
313
|
-
else:
|
|
314
|
-
data = json.loads(Path(args.input).read_text())
|
|
315
|
-
|
|
316
|
-
html_output = generate_html(data, skill_name=args.skill_name)
|
|
317
|
-
|
|
318
|
-
if args.output:
|
|
319
|
-
Path(args.output).write_text(html_output)
|
|
320
|
-
print(f"Report written to {args.output}", file=sys.stderr)
|
|
321
|
-
else:
|
|
322
|
-
print(html_output)
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
if __name__ == "__main__":
|
|
326
|
-
main()
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Generate an HTML report from run_loop.py output.
|
|
3
|
+
|
|
4
|
+
Takes the JSON output from run_loop.py and generates a visual HTML report
|
|
5
|
+
showing each description attempt with check/x for each test case.
|
|
6
|
+
Distinguishes between train and test queries.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import html
|
|
11
|
+
import json
|
|
12
|
+
import sys
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def generate_html(data: dict, auto_refresh: bool = False, skill_name: str = "") -> str:
|
|
17
|
+
"""Generate HTML report from loop output data. If auto_refresh is True, adds a meta refresh tag."""
|
|
18
|
+
history = data.get("history", [])
|
|
19
|
+
holdout = data.get("holdout", 0)
|
|
20
|
+
title_prefix = html.escape(skill_name + " \u2014 ") if skill_name else ""
|
|
21
|
+
|
|
22
|
+
# Get all unique queries from train and test sets, with should_trigger info
|
|
23
|
+
train_queries: list[dict] = []
|
|
24
|
+
test_queries: list[dict] = []
|
|
25
|
+
if history:
|
|
26
|
+
for r in history[0].get("train_results", history[0].get("results", [])):
|
|
27
|
+
train_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)})
|
|
28
|
+
if history[0].get("test_results"):
|
|
29
|
+
for r in history[0].get("test_results", []):
|
|
30
|
+
test_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)})
|
|
31
|
+
|
|
32
|
+
refresh_tag = ' <meta http-equiv="refresh" content="5">\n' if auto_refresh else ""
|
|
33
|
+
|
|
34
|
+
html_parts = ["""<!DOCTYPE html>
|
|
35
|
+
<html>
|
|
36
|
+
<head>
|
|
37
|
+
<meta charset="utf-8">
|
|
38
|
+
""" + refresh_tag + """ <title>""" + title_prefix + """Skill Description Optimization</title>
|
|
39
|
+
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
40
|
+
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
41
|
+
<link href="https://fonts.googleapis.com/css2?family=Poppins:wght@500;600&family=Lora:wght@400;500&display=swap" rel="stylesheet">
|
|
42
|
+
<style>
|
|
43
|
+
body {
|
|
44
|
+
font-family: 'Lora', Georgia, serif;
|
|
45
|
+
max-width: 100%;
|
|
46
|
+
margin: 0 auto;
|
|
47
|
+
padding: 20px;
|
|
48
|
+
background: #faf9f5;
|
|
49
|
+
color: #141413;
|
|
50
|
+
}
|
|
51
|
+
h1 { font-family: 'Poppins', sans-serif; color: #141413; }
|
|
52
|
+
.explainer {
|
|
53
|
+
background: white;
|
|
54
|
+
padding: 15px;
|
|
55
|
+
border-radius: 6px;
|
|
56
|
+
margin-bottom: 20px;
|
|
57
|
+
border: 1px solid #e8e6dc;
|
|
58
|
+
color: #b0aea5;
|
|
59
|
+
font-size: 0.875rem;
|
|
60
|
+
line-height: 1.6;
|
|
61
|
+
}
|
|
62
|
+
.summary {
|
|
63
|
+
background: white;
|
|
64
|
+
padding: 15px;
|
|
65
|
+
border-radius: 6px;
|
|
66
|
+
margin-bottom: 20px;
|
|
67
|
+
border: 1px solid #e8e6dc;
|
|
68
|
+
}
|
|
69
|
+
.summary p { margin: 5px 0; }
|
|
70
|
+
.best { color: #788c5d; font-weight: bold; }
|
|
71
|
+
.table-container {
|
|
72
|
+
overflow-x: auto;
|
|
73
|
+
width: 100%;
|
|
74
|
+
}
|
|
75
|
+
table {
|
|
76
|
+
border-collapse: collapse;
|
|
77
|
+
background: white;
|
|
78
|
+
border: 1px solid #e8e6dc;
|
|
79
|
+
border-radius: 6px;
|
|
80
|
+
font-size: 12px;
|
|
81
|
+
min-width: 100%;
|
|
82
|
+
}
|
|
83
|
+
th, td {
|
|
84
|
+
padding: 8px;
|
|
85
|
+
text-align: left;
|
|
86
|
+
border: 1px solid #e8e6dc;
|
|
87
|
+
white-space: normal;
|
|
88
|
+
word-wrap: break-word;
|
|
89
|
+
}
|
|
90
|
+
th {
|
|
91
|
+
font-family: 'Poppins', sans-serif;
|
|
92
|
+
background: #141413;
|
|
93
|
+
color: #faf9f5;
|
|
94
|
+
font-weight: 500;
|
|
95
|
+
}
|
|
96
|
+
th.test-col {
|
|
97
|
+
background: #6a9bcc;
|
|
98
|
+
}
|
|
99
|
+
th.query-col { min-width: 200px; }
|
|
100
|
+
td.description {
|
|
101
|
+
font-family: monospace;
|
|
102
|
+
font-size: 11px;
|
|
103
|
+
word-wrap: break-word;
|
|
104
|
+
max-width: 400px;
|
|
105
|
+
}
|
|
106
|
+
td.result {
|
|
107
|
+
text-align: center;
|
|
108
|
+
font-size: 16px;
|
|
109
|
+
min-width: 40px;
|
|
110
|
+
}
|
|
111
|
+
td.test-result {
|
|
112
|
+
background: #f0f6fc;
|
|
113
|
+
}
|
|
114
|
+
.pass { color: #788c5d; }
|
|
115
|
+
.fail { color: #c44; }
|
|
116
|
+
.rate {
|
|
117
|
+
font-size: 9px;
|
|
118
|
+
color: #b0aea5;
|
|
119
|
+
display: block;
|
|
120
|
+
}
|
|
121
|
+
tr:hover { background: #faf9f5; }
|
|
122
|
+
.score {
|
|
123
|
+
display: inline-block;
|
|
124
|
+
padding: 2px 6px;
|
|
125
|
+
border-radius: 4px;
|
|
126
|
+
font-weight: bold;
|
|
127
|
+
font-size: 11px;
|
|
128
|
+
}
|
|
129
|
+
.score-good { background: #eef2e8; color: #788c5d; }
|
|
130
|
+
.score-ok { background: #fef3c7; color: #d97706; }
|
|
131
|
+
.score-bad { background: #fceaea; color: #c44; }
|
|
132
|
+
.train-label { color: #b0aea5; font-size: 10px; }
|
|
133
|
+
.test-label { color: #6a9bcc; font-size: 10px; font-weight: bold; }
|
|
134
|
+
.best-row { background: #f5f8f2; }
|
|
135
|
+
th.positive-col { border-bottom: 3px solid #788c5d; }
|
|
136
|
+
th.negative-col { border-bottom: 3px solid #c44; }
|
|
137
|
+
th.test-col.positive-col { border-bottom: 3px solid #788c5d; }
|
|
138
|
+
th.test-col.negative-col { border-bottom: 3px solid #c44; }
|
|
139
|
+
.legend { font-family: 'Poppins', sans-serif; display: flex; gap: 20px; margin-bottom: 10px; font-size: 13px; align-items: center; }
|
|
140
|
+
.legend-item { display: flex; align-items: center; gap: 6px; }
|
|
141
|
+
.legend-swatch { width: 16px; height: 16px; border-radius: 3px; display: inline-block; }
|
|
142
|
+
.swatch-positive { background: #141413; border-bottom: 3px solid #788c5d; }
|
|
143
|
+
.swatch-negative { background: #141413; border-bottom: 3px solid #c44; }
|
|
144
|
+
.swatch-test { background: #6a9bcc; }
|
|
145
|
+
.swatch-train { background: #141413; }
|
|
146
|
+
</style>
|
|
147
|
+
</head>
|
|
148
|
+
<body>
|
|
149
|
+
<h1>""" + title_prefix + """Skill Description Optimization</h1>
|
|
150
|
+
<div class="explainer">
|
|
151
|
+
<strong>Optimizing your skill's description.</strong> This page updates automatically as Claude tests different versions of your skill's description. Each row is an iteration — a new description attempt. The columns show test queries: green checkmarks mean the skill triggered correctly (or correctly didn't trigger), red crosses mean it got it wrong. The "Train" score shows performance on queries used to improve the description; the "Test" score shows performance on held-out queries the optimizer hasn't seen. When it's done, Claude will apply the best-performing description to your skill.
|
|
152
|
+
</div>
|
|
153
|
+
"""]
|
|
154
|
+
|
|
155
|
+
# Summary section
|
|
156
|
+
best_test_score = data.get('best_test_score')
|
|
157
|
+
best_train_score = data.get('best_train_score')
|
|
158
|
+
html_parts.append(f"""
|
|
159
|
+
<div class="summary">
|
|
160
|
+
<p><strong>Original:</strong> {html.escape(data.get('original_description', 'N/A'))}</p>
|
|
161
|
+
<p class="best"><strong>Best:</strong> {html.escape(data.get('best_description', 'N/A'))}</p>
|
|
162
|
+
<p><strong>Best Score:</strong> {data.get('best_score', 'N/A')} {'(test)' if best_test_score else '(train)'}</p>
|
|
163
|
+
<p><strong>Iterations:</strong> {data.get('iterations_run', 0)} | <strong>Train:</strong> {data.get('train_size', '?')} | <strong>Test:</strong> {data.get('test_size', '?')}</p>
|
|
164
|
+
</div>
|
|
165
|
+
""")
|
|
166
|
+
|
|
167
|
+
# Legend
|
|
168
|
+
html_parts.append("""
|
|
169
|
+
<div class="legend">
|
|
170
|
+
<span style="font-weight:600">Query columns:</span>
|
|
171
|
+
<span class="legend-item"><span class="legend-swatch swatch-positive"></span> Should trigger</span>
|
|
172
|
+
<span class="legend-item"><span class="legend-swatch swatch-negative"></span> Should NOT trigger</span>
|
|
173
|
+
<span class="legend-item"><span class="legend-swatch swatch-train"></span> Train</span>
|
|
174
|
+
<span class="legend-item"><span class="legend-swatch swatch-test"></span> Test</span>
|
|
175
|
+
</div>
|
|
176
|
+
""")
|
|
177
|
+
|
|
178
|
+
# Table header
|
|
179
|
+
html_parts.append("""
|
|
180
|
+
<div class="table-container">
|
|
181
|
+
<table>
|
|
182
|
+
<thead>
|
|
183
|
+
<tr>
|
|
184
|
+
<th>Iter</th>
|
|
185
|
+
<th>Train</th>
|
|
186
|
+
<th>Test</th>
|
|
187
|
+
<th class="query-col">Description</th>
|
|
188
|
+
""")
|
|
189
|
+
|
|
190
|
+
# Add column headers for train queries
|
|
191
|
+
for qinfo in train_queries:
|
|
192
|
+
polarity = "positive-col" if qinfo["should_trigger"] else "negative-col"
|
|
193
|
+
html_parts.append(f' <th class="{polarity}">{html.escape(qinfo["query"])}</th>\n')
|
|
194
|
+
|
|
195
|
+
# Add column headers for test queries (different color)
|
|
196
|
+
for qinfo in test_queries:
|
|
197
|
+
polarity = "positive-col" if qinfo["should_trigger"] else "negative-col"
|
|
198
|
+
html_parts.append(f' <th class="test-col {polarity}">{html.escape(qinfo["query"])}</th>\n')
|
|
199
|
+
|
|
200
|
+
html_parts.append(""" </tr>
|
|
201
|
+
</thead>
|
|
202
|
+
<tbody>
|
|
203
|
+
""")
|
|
204
|
+
|
|
205
|
+
# Find best iteration for highlighting
|
|
206
|
+
if test_queries:
|
|
207
|
+
best_iter = max(history, key=lambda h: h.get("test_passed") or 0).get("iteration")
|
|
208
|
+
else:
|
|
209
|
+
best_iter = max(history, key=lambda h: h.get("train_passed", h.get("passed", 0))).get("iteration")
|
|
210
|
+
|
|
211
|
+
# Add rows for each iteration
|
|
212
|
+
for h in history:
|
|
213
|
+
iteration = h.get("iteration", "?")
|
|
214
|
+
train_passed = h.get("train_passed", h.get("passed", 0))
|
|
215
|
+
train_total = h.get("train_total", h.get("total", 0))
|
|
216
|
+
test_passed = h.get("test_passed")
|
|
217
|
+
test_total = h.get("test_total")
|
|
218
|
+
description = h.get("description", "")
|
|
219
|
+
train_results = h.get("train_results", h.get("results", []))
|
|
220
|
+
test_results = h.get("test_results", [])
|
|
221
|
+
|
|
222
|
+
# Create lookups for results by query
|
|
223
|
+
train_by_query = {r["query"]: r for r in train_results}
|
|
224
|
+
test_by_query = {r["query"]: r for r in test_results} if test_results else {}
|
|
225
|
+
|
|
226
|
+
# Compute aggregate correct/total runs across all retries
|
|
227
|
+
def aggregate_runs(results: list[dict]) -> tuple[int, int]:
|
|
228
|
+
correct = 0
|
|
229
|
+
total = 0
|
|
230
|
+
for r in results:
|
|
231
|
+
runs = r.get("runs", 0)
|
|
232
|
+
triggers = r.get("triggers", 0)
|
|
233
|
+
total += runs
|
|
234
|
+
if r.get("should_trigger", True):
|
|
235
|
+
correct += triggers
|
|
236
|
+
else:
|
|
237
|
+
correct += runs - triggers
|
|
238
|
+
return correct, total
|
|
239
|
+
|
|
240
|
+
train_correct, train_runs = aggregate_runs(train_results)
|
|
241
|
+
test_correct, test_runs = aggregate_runs(test_results)
|
|
242
|
+
|
|
243
|
+
# Determine score classes
|
|
244
|
+
def score_class(correct: int, total: int) -> str:
|
|
245
|
+
if total > 0:
|
|
246
|
+
ratio = correct / total
|
|
247
|
+
if ratio >= 0.8:
|
|
248
|
+
return "score-good"
|
|
249
|
+
elif ratio >= 0.5:
|
|
250
|
+
return "score-ok"
|
|
251
|
+
return "score-bad"
|
|
252
|
+
|
|
253
|
+
train_class = score_class(train_correct, train_runs)
|
|
254
|
+
test_class = score_class(test_correct, test_runs)
|
|
255
|
+
|
|
256
|
+
row_class = "best-row" if iteration == best_iter else ""
|
|
257
|
+
|
|
258
|
+
html_parts.append(f""" <tr class="{row_class}">
|
|
259
|
+
<td>{iteration}</td>
|
|
260
|
+
<td><span class="score {train_class}">{train_correct}/{train_runs}</span></td>
|
|
261
|
+
<td><span class="score {test_class}">{test_correct}/{test_runs}</span></td>
|
|
262
|
+
<td class="description">{html.escape(description)}</td>
|
|
263
|
+
""")
|
|
264
|
+
|
|
265
|
+
# Add result for each train query
|
|
266
|
+
for qinfo in train_queries:
|
|
267
|
+
r = train_by_query.get(qinfo["query"], {})
|
|
268
|
+
did_pass = r.get("pass", False)
|
|
269
|
+
triggers = r.get("triggers", 0)
|
|
270
|
+
runs = r.get("runs", 0)
|
|
271
|
+
|
|
272
|
+
icon = "✓" if did_pass else "✗"
|
|
273
|
+
css_class = "pass" if did_pass else "fail"
|
|
274
|
+
|
|
275
|
+
html_parts.append(f' <td class="result {css_class}">{icon}<span class="rate">{triggers}/{runs}</span></td>\n')
|
|
276
|
+
|
|
277
|
+
# Add result for each test query (with different background)
|
|
278
|
+
for qinfo in test_queries:
|
|
279
|
+
r = test_by_query.get(qinfo["query"], {})
|
|
280
|
+
did_pass = r.get("pass", False)
|
|
281
|
+
triggers = r.get("triggers", 0)
|
|
282
|
+
runs = r.get("runs", 0)
|
|
283
|
+
|
|
284
|
+
icon = "✓" if did_pass else "✗"
|
|
285
|
+
css_class = "pass" if did_pass else "fail"
|
|
286
|
+
|
|
287
|
+
html_parts.append(f' <td class="result test-result {css_class}">{icon}<span class="rate">{triggers}/{runs}</span></td>\n')
|
|
288
|
+
|
|
289
|
+
html_parts.append(" </tr>\n")
|
|
290
|
+
|
|
291
|
+
html_parts.append(""" </tbody>
|
|
292
|
+
</table>
|
|
293
|
+
</div>
|
|
294
|
+
""")
|
|
295
|
+
|
|
296
|
+
html_parts.append("""
|
|
297
|
+
</body>
|
|
298
|
+
</html>
|
|
299
|
+
""")
|
|
300
|
+
|
|
301
|
+
return "".join(html_parts)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def main():
|
|
305
|
+
parser = argparse.ArgumentParser(description="Generate HTML report from run_loop output")
|
|
306
|
+
parser.add_argument("input", help="Path to JSON output from run_loop.py (or - for stdin)")
|
|
307
|
+
parser.add_argument("-o", "--output", default=None, help="Output HTML file (default: stdout)")
|
|
308
|
+
parser.add_argument("--skill-name", default="", help="Skill name to include in the report title")
|
|
309
|
+
args = parser.parse_args()
|
|
310
|
+
|
|
311
|
+
if args.input == "-":
|
|
312
|
+
data = json.load(sys.stdin)
|
|
313
|
+
else:
|
|
314
|
+
data = json.loads(Path(args.input).read_text())
|
|
315
|
+
|
|
316
|
+
html_output = generate_html(data, skill_name=args.skill_name)
|
|
317
|
+
|
|
318
|
+
if args.output:
|
|
319
|
+
Path(args.output).write_text(html_output)
|
|
320
|
+
print(f"Report written to {args.output}", file=sys.stderr)
|
|
321
|
+
else:
|
|
322
|
+
print(html_output)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
if __name__ == "__main__":
|
|
326
|
+
main()
|