ai-engineering-init 1.7.0 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/.claude/hooks/skill-forced-eval.js +46 -62
  2. package/.claude/settings.json +10 -1
  3. package/.claude/skills/api-development/SKILL.md +179 -130
  4. package/.claude/skills/architecture-design/SKILL.md +102 -212
  5. package/.claude/skills/backend-annotations/SKILL.md +166 -220
  6. package/.claude/skills/bug-detective/SKILL.md +225 -186
  7. package/.claude/skills/code-patterns/SKILL.md +127 -244
  8. package/.claude/skills/collaborating-with-codex/SKILL.md +96 -113
  9. package/.claude/skills/crud-development/SKILL.md +226 -307
  10. package/.claude/skills/data-permission/SKILL.md +131 -202
  11. package/.claude/skills/database-ops/SKILL.md +158 -355
  12. package/.claude/skills/error-handler/SKILL.md +224 -285
  13. package/.claude/skills/file-oss-management/SKILL.md +174 -169
  14. package/.claude/skills/git-workflow/SKILL.md +123 -341
  15. package/.claude/skills/json-serialization/SKILL.md +121 -137
  16. package/.claude/skills/performance-doctor/SKILL.md +83 -89
  17. package/.claude/skills/redis-cache/SKILL.md +134 -185
  18. package/.claude/skills/scheduled-jobs/SKILL.md +187 -224
  19. package/.claude/skills/security-guard/SKILL.md +168 -276
  20. package/.claude/skills/sms-mail/SKILL.md +266 -228
  21. package/.claude/skills/social-login/SKILL.md +257 -195
  22. package/.claude/skills/tenant-management/SKILL.md +172 -188
  23. package/.claude/skills/utils-toolkit/SKILL.md +214 -222
  24. package/.claude/skills/websocket-sse/SKILL.md +251 -172
  25. package/.claude/skills/workflow-engine/SKILL.md +178 -250
  26. package/.codex/skills/api-development/SKILL.md +179 -130
  27. package/.codex/skills/architecture-design/SKILL.md +102 -212
  28. package/.codex/skills/backend-annotations/SKILL.md +166 -220
  29. package/.codex/skills/bug-detective/SKILL.md +225 -186
  30. package/.codex/skills/code-patterns/SKILL.md +127 -244
  31. package/.codex/skills/collaborating-with-codex/SKILL.md +96 -113
  32. package/.codex/skills/crud-development/SKILL.md +226 -307
  33. package/.codex/skills/data-permission/SKILL.md +131 -202
  34. package/.codex/skills/database-ops/SKILL.md +158 -355
  35. package/.codex/skills/error-handler/SKILL.md +224 -285
  36. package/.codex/skills/file-oss-management/SKILL.md +174 -169
  37. package/.codex/skills/git-workflow/SKILL.md +123 -341
  38. package/.codex/skills/json-serialization/SKILL.md +121 -137
  39. package/.codex/skills/performance-doctor/SKILL.md +83 -89
  40. package/.codex/skills/redis-cache/SKILL.md +134 -185
  41. package/.codex/skills/scheduled-jobs/SKILL.md +187 -224
  42. package/.codex/skills/security-guard/SKILL.md +168 -276
  43. package/.codex/skills/sms-mail/SKILL.md +266 -228
  44. package/.codex/skills/social-login/SKILL.md +257 -195
  45. package/.codex/skills/tenant-management/SKILL.md +172 -188
  46. package/.codex/skills/utils-toolkit/SKILL.md +214 -222
  47. package/.codex/skills/websocket-sse/SKILL.md +251 -172
  48. package/.codex/skills/workflow-engine/SKILL.md +178 -250
  49. package/.cursor/hooks/cursor-skill-eval.js +66 -6
  50. package/.cursor/skills/api-development/SKILL.md +179 -130
  51. package/.cursor/skills/architecture-design/SKILL.md +102 -212
  52. package/.cursor/skills/backend-annotations/SKILL.md +166 -220
  53. package/.cursor/skills/bug-detective/SKILL.md +225 -186
  54. package/.cursor/skills/code-patterns/SKILL.md +127 -244
  55. package/.cursor/skills/collaborating-with-codex/SKILL.md +96 -113
  56. package/.cursor/skills/crud-development/SKILL.md +226 -307
  57. package/.cursor/skills/data-permission/SKILL.md +131 -202
  58. package/.cursor/skills/database-ops/SKILL.md +158 -355
  59. package/.cursor/skills/error-handler/SKILL.md +224 -285
  60. package/.cursor/skills/file-oss-management/SKILL.md +174 -169
  61. package/.cursor/skills/git-workflow/SKILL.md +123 -341
  62. package/.cursor/skills/json-serialization/SKILL.md +121 -137
  63. package/.cursor/skills/performance-doctor/SKILL.md +83 -89
  64. package/.cursor/skills/redis-cache/SKILL.md +134 -185
  65. package/.cursor/skills/scheduled-jobs/SKILL.md +187 -224
  66. package/.cursor/skills/security-guard/SKILL.md +168 -276
  67. package/.cursor/skills/sms-mail/SKILL.md +266 -228
  68. package/.cursor/skills/social-login/SKILL.md +257 -195
  69. package/.cursor/skills/tenant-management/SKILL.md +172 -188
  70. package/.cursor/skills/utils-toolkit/SKILL.md +214 -222
  71. package/.cursor/skills/websocket-sse/SKILL.md +251 -172
  72. package/.cursor/skills/workflow-engine/SKILL.md +178 -250
  73. package/AGENTS.md +49 -540
  74. package/CLAUDE.md +73 -119
  75. package/README.md +37 -6
  76. package/bin/index.js +5 -1
  77. package/package.json +1 -1
  78. package/src/skills/api-development/SKILL.md +179 -130
  79. package/src/skills/architecture-design/SKILL.md +102 -212
  80. package/src/skills/backend-annotations/SKILL.md +166 -220
  81. package/src/skills/bug-detective/SKILL.md +225 -186
  82. package/src/skills/code-patterns/SKILL.md +127 -244
  83. package/src/skills/collaborating-with-codex/SKILL.md +96 -113
  84. package/src/skills/crud-development/SKILL.md +226 -307
  85. package/src/skills/data-permission/SKILL.md +131 -202
  86. package/src/skills/database-ops/SKILL.md +158 -355
  87. package/src/skills/error-handler/SKILL.md +224 -285
  88. package/src/skills/file-oss-management/SKILL.md +174 -169
  89. package/src/skills/git-workflow/SKILL.md +123 -341
  90. package/src/skills/json-serialization/SKILL.md +121 -137
  91. package/src/skills/performance-doctor/SKILL.md +83 -89
  92. package/src/skills/redis-cache/SKILL.md +134 -185
  93. package/src/skills/scheduled-jobs/SKILL.md +187 -224
  94. package/src/skills/security-guard/SKILL.md +168 -276
  95. package/src/skills/sms-mail/SKILL.md +266 -228
  96. package/src/skills/social-login/SKILL.md +257 -195
  97. package/src/skills/tenant-management/SKILL.md +172 -188
  98. package/src/skills/utils-toolkit/SKILL.md +214 -222
  99. package/src/skills/websocket-sse/SKILL.md +251 -172
  100. package/src/skills/workflow-engine/SKILL.md +178 -250
  101. package/.claude/skills/skill-creator/LICENSE.txt +0 -202
  102. package/.claude/skills/skill-creator/SKILL.md +0 -479
  103. package/.claude/skills/skill-creator/agents/analyzer.md +0 -274
  104. package/.claude/skills/skill-creator/agents/comparator.md +0 -202
  105. package/.claude/skills/skill-creator/agents/grader.md +0 -223
  106. package/.claude/skills/skill-creator/assets/eval_review.html +0 -146
  107. package/.claude/skills/skill-creator/eval-viewer/generate_review.py +0 -471
  108. package/.claude/skills/skill-creator/eval-viewer/viewer.html +0 -1325
  109. package/.claude/skills/skill-creator/references/schemas.md +0 -430
  110. package/.claude/skills/skill-creator/scripts/__init__.py +0 -0
  111. package/.claude/skills/skill-creator/scripts/aggregate_benchmark.py +0 -401
  112. package/.claude/skills/skill-creator/scripts/generate_report.py +0 -326
  113. package/.claude/skills/skill-creator/scripts/improve_description.py +0 -248
  114. package/.claude/skills/skill-creator/scripts/package_skill.py +0 -136
  115. package/.claude/skills/skill-creator/scripts/quick_validate.py +0 -103
  116. package/.claude/skills/skill-creator/scripts/run_eval.py +0 -310
  117. package/.claude/skills/skill-creator/scripts/run_loop.py +0 -332
  118. package/.claude/skills/skill-creator/scripts/utils.py +0 -47
@@ -1,146 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="en">
3
- <head>
4
- <meta charset="UTF-8">
5
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
- <title>Eval Set Review - __SKILL_NAME_PLACEHOLDER__</title>
7
- <link rel="preconnect" href="https://fonts.googleapis.com">
8
- <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
9
- <link href="https://fonts.googleapis.com/css2?family=Poppins:wght@500;600&family=Lora:wght@400;500&display=swap" rel="stylesheet">
10
- <style>
11
- * { box-sizing: border-box; margin: 0; padding: 0; }
12
- body { font-family: 'Lora', Georgia, serif; background: #faf9f5; padding: 2rem; color: #141413; }
13
- h1 { font-family: 'Poppins', sans-serif; margin-bottom: 0.5rem; font-size: 1.5rem; }
14
- .description { color: #b0aea5; margin-bottom: 1.5rem; font-style: italic; max-width: 900px; }
15
- .controls { margin-bottom: 1rem; display: flex; gap: 0.5rem; }
16
- .btn { font-family: 'Poppins', sans-serif; padding: 0.5rem 1rem; border: none; border-radius: 6px; cursor: pointer; font-size: 0.875rem; font-weight: 500; }
17
- .btn-add { background: #6a9bcc; color: white; }
18
- .btn-add:hover { background: #5889b8; }
19
- .btn-export { background: #d97757; color: white; }
20
- .btn-export:hover { background: #c4613f; }
21
- table { width: 100%; max-width: 1100px; border-collapse: collapse; background: white; border-radius: 6px; overflow: hidden; box-shadow: 0 1px 3px rgba(0,0,0,0.08); }
22
- th { font-family: 'Poppins', sans-serif; background: #141413; color: #faf9f5; padding: 0.75rem 1rem; text-align: left; font-size: 0.875rem; }
23
- td { padding: 0.75rem 1rem; border-bottom: 1px solid #e8e6dc; vertical-align: top; }
24
- tr:nth-child(even) td { background: #faf9f5; }
25
- tr:hover td { background: #f3f1ea; }
26
- .section-header td { background: #e8e6dc; font-family: 'Poppins', sans-serif; font-weight: 500; font-size: 0.8rem; color: #141413; text-transform: uppercase; letter-spacing: 0.05em; }
27
- .query-input { width: 100%; padding: 0.4rem; border: 1px solid #e8e6dc; border-radius: 4px; font-size: 0.875rem; font-family: 'Lora', Georgia, serif; resize: vertical; min-height: 60px; }
28
- .query-input:focus { outline: none; border-color: #d97757; box-shadow: 0 0 0 2px rgba(217,119,87,0.15); }
29
- .toggle { position: relative; display: inline-block; width: 44px; height: 24px; }
30
- .toggle input { opacity: 0; width: 0; height: 0; }
31
- .toggle .slider { position: absolute; inset: 0; background: #b0aea5; border-radius: 24px; cursor: pointer; transition: 0.2s; }
32
- .toggle .slider::before { content: ""; position: absolute; width: 18px; height: 18px; left: 3px; bottom: 3px; background: white; border-radius: 50%; transition: 0.2s; }
33
- .toggle input:checked + .slider { background: #d97757; }
34
- .toggle input:checked + .slider::before { transform: translateX(20px); }
35
- .btn-delete { background: #c44; color: white; padding: 0.3rem 0.6rem; border: none; border-radius: 4px; cursor: pointer; font-size: 0.75rem; font-family: 'Poppins', sans-serif; }
36
- .btn-delete:hover { background: #a33; }
37
- .summary { margin-top: 1rem; color: #b0aea5; font-size: 0.875rem; }
38
- </style>
39
- </head>
40
- <body>
41
- <h1>Eval Set Review: <span id="skill-name">__SKILL_NAME_PLACEHOLDER__</span></h1>
42
- <p class="description">Current description: <span id="skill-desc">__SKILL_DESCRIPTION_PLACEHOLDER__</span></p>
43
-
44
- <div class="controls">
45
- <button class="btn btn-add" onclick="addRow()">+ Add Query</button>
46
- <button class="btn btn-export" onclick="exportEvalSet()">Export Eval Set</button>
47
- </div>
48
-
49
- <table>
50
- <thead>
51
- <tr>
52
- <th style="width:65%">Query</th>
53
- <th style="width:18%">Should Trigger</th>
54
- <th style="width:10%">Actions</th>
55
- </tr>
56
- </thead>
57
- <tbody id="eval-body"></tbody>
58
- </table>
59
-
60
- <p class="summary" id="summary"></p>
61
-
62
- <script>
63
- const EVAL_DATA = __EVAL_DATA_PLACEHOLDER__;
64
-
65
- let evalItems = [...EVAL_DATA];
66
-
67
- function render() {
68
- const tbody = document.getElementById('eval-body');
69
- tbody.innerHTML = '';
70
-
71
- // Sort: should-trigger first, then should-not-trigger
72
- const sorted = evalItems
73
- .map((item, origIdx) => ({ ...item, origIdx }))
74
- .sort((a, b) => (b.should_trigger ? 1 : 0) - (a.should_trigger ? 1 : 0));
75
-
76
- let lastGroup = null;
77
- sorted.forEach(item => {
78
- const group = item.should_trigger ? 'trigger' : 'no-trigger';
79
- if (group !== lastGroup) {
80
- const headerRow = document.createElement('tr');
81
- headerRow.className = 'section-header';
82
- headerRow.innerHTML = `<td colspan="3">${item.should_trigger ? 'Should Trigger' : 'Should NOT Trigger'}</td>`;
83
- tbody.appendChild(headerRow);
84
- lastGroup = group;
85
- }
86
-
87
- const idx = item.origIdx;
88
- const tr = document.createElement('tr');
89
- tr.innerHTML = `
90
- <td><textarea class="query-input" onchange="updateQuery(${idx}, this.value)">${escapeHtml(item.query)}</textarea></td>
91
- <td>
92
- <label class="toggle">
93
- <input type="checkbox" ${item.should_trigger ? 'checked' : ''} onchange="updateTrigger(${idx}, this.checked)">
94
- <span class="slider"></span>
95
- </label>
96
- <span style="margin-left:8px;font-size:0.8rem;color:#b0aea5">${item.should_trigger ? 'Yes' : 'No'}</span>
97
- </td>
98
- <td><button class="btn-delete" onclick="deleteRow(${idx})">Delete</button></td>
99
- `;
100
- tbody.appendChild(tr);
101
- });
102
- updateSummary();
103
- }
104
-
105
- function escapeHtml(text) {
106
- const div = document.createElement('div');
107
- div.textContent = text;
108
- return div.innerHTML;
109
- }
110
-
111
- function updateQuery(idx, value) { evalItems[idx].query = value; updateSummary(); }
112
- function updateTrigger(idx, value) { evalItems[idx].should_trigger = value; render(); }
113
- function deleteRow(idx) { evalItems.splice(idx, 1); render(); }
114
-
115
- function addRow() {
116
- evalItems.push({ query: '', should_trigger: true });
117
- render();
118
- const inputs = document.querySelectorAll('.query-input');
119
- inputs[inputs.length - 1].focus();
120
- }
121
-
122
- function updateSummary() {
123
- const trigger = evalItems.filter(i => i.should_trigger).length;
124
- const noTrigger = evalItems.filter(i => !i.should_trigger).length;
125
- document.getElementById('summary').textContent =
126
- `${evalItems.length} queries total: ${trigger} should trigger, ${noTrigger} should not trigger`;
127
- }
128
-
129
- function exportEvalSet() {
130
- const valid = evalItems.filter(i => i.query.trim() !== '');
131
- const data = valid.map(i => ({ query: i.query.trim(), should_trigger: i.should_trigger }));
132
- const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' });
133
- const url = URL.createObjectURL(blob);
134
- const a = document.createElement('a');
135
- a.href = url;
136
- a.download = 'eval_set.json';
137
- document.body.appendChild(a);
138
- a.click();
139
- document.body.removeChild(a);
140
- URL.revokeObjectURL(url);
141
- }
142
-
143
- render();
144
- </script>
145
- </body>
146
- </html>
@@ -1,471 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Generate and serve a review page for eval results.
3
-
4
- Reads the workspace directory, discovers runs (directories with outputs/),
5
- embeds all output data into a self-contained HTML page, and serves it via
6
- a tiny HTTP server. Feedback auto-saves to feedback.json in the workspace.
7
-
8
- Usage:
9
- python generate_review.py <workspace-path> [--port PORT] [--skill-name NAME]
10
- python generate_review.py <workspace-path> --previous-feedback /path/to/old/feedback.json
11
-
12
- No dependencies beyond the Python stdlib are required.
13
- """
14
-
15
- import argparse
16
- import base64
17
- import json
18
- import mimetypes
19
- import os
20
- import re
21
- import signal
22
- import subprocess
23
- import sys
24
- import time
25
- import webbrowser
26
- from functools import partial
27
- from http.server import HTTPServer, BaseHTTPRequestHandler
28
- from pathlib import Path
29
-
30
- # Files to exclude from output listings
31
- METADATA_FILES = {"transcript.md", "user_notes.md", "metrics.json"}
32
-
33
- # Extensions we render as inline text
34
- TEXT_EXTENSIONS = {
35
- ".txt", ".md", ".json", ".csv", ".py", ".js", ".ts", ".tsx", ".jsx",
36
- ".yaml", ".yml", ".xml", ".html", ".css", ".sh", ".rb", ".go", ".rs",
37
- ".java", ".c", ".cpp", ".h", ".hpp", ".sql", ".r", ".toml",
38
- }
39
-
40
- # Extensions we render as inline images
41
- IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp"}
42
-
43
- # MIME type overrides for common types
44
- MIME_OVERRIDES = {
45
- ".svg": "image/svg+xml",
46
- ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
47
- ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
48
- ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
49
- }
50
-
51
-
52
- def get_mime_type(path: Path) -> str:
53
- ext = path.suffix.lower()
54
- if ext in MIME_OVERRIDES:
55
- return MIME_OVERRIDES[ext]
56
- mime, _ = mimetypes.guess_type(str(path))
57
- return mime or "application/octet-stream"
58
-
59
-
60
- def find_runs(workspace: Path) -> list[dict]:
61
- """Recursively find directories that contain an outputs/ subdirectory."""
62
- runs: list[dict] = []
63
- _find_runs_recursive(workspace, workspace, runs)
64
- runs.sort(key=lambda r: (r.get("eval_id", float("inf")), r["id"]))
65
- return runs
66
-
67
-
68
- def _find_runs_recursive(root: Path, current: Path, runs: list[dict]) -> None:
69
- if not current.is_dir():
70
- return
71
-
72
- outputs_dir = current / "outputs"
73
- if outputs_dir.is_dir():
74
- run = build_run(root, current)
75
- if run:
76
- runs.append(run)
77
- return
78
-
79
- skip = {"node_modules", ".git", "__pycache__", "skill", "inputs"}
80
- for child in sorted(current.iterdir()):
81
- if child.is_dir() and child.name not in skip:
82
- _find_runs_recursive(root, child, runs)
83
-
84
-
85
- def build_run(root: Path, run_dir: Path) -> dict | None:
86
- """Build a run dict with prompt, outputs, and grading data."""
87
- prompt = ""
88
- eval_id = None
89
-
90
- # Try eval_metadata.json
91
- for candidate in [run_dir / "eval_metadata.json", run_dir.parent / "eval_metadata.json"]:
92
- if candidate.exists():
93
- try:
94
- metadata = json.loads(candidate.read_text())
95
- prompt = metadata.get("prompt", "")
96
- eval_id = metadata.get("eval_id")
97
- except (json.JSONDecodeError, OSError):
98
- pass
99
- if prompt:
100
- break
101
-
102
- # Fall back to transcript.md
103
- if not prompt:
104
- for candidate in [run_dir / "transcript.md", run_dir / "outputs" / "transcript.md"]:
105
- if candidate.exists():
106
- try:
107
- text = candidate.read_text()
108
- match = re.search(r"## Eval Prompt\n\n([\s\S]*?)(?=\n##|$)", text)
109
- if match:
110
- prompt = match.group(1).strip()
111
- except OSError:
112
- pass
113
- if prompt:
114
- break
115
-
116
- if not prompt:
117
- prompt = "(No prompt found)"
118
-
119
- run_id = str(run_dir.relative_to(root)).replace("/", "-").replace("\\", "-")
120
-
121
- # Collect output files
122
- outputs_dir = run_dir / "outputs"
123
- output_files: list[dict] = []
124
- if outputs_dir.is_dir():
125
- for f in sorted(outputs_dir.iterdir()):
126
- if f.is_file() and f.name not in METADATA_FILES:
127
- output_files.append(embed_file(f))
128
-
129
- # Load grading if present
130
- grading = None
131
- for candidate in [run_dir / "grading.json", run_dir.parent / "grading.json"]:
132
- if candidate.exists():
133
- try:
134
- grading = json.loads(candidate.read_text())
135
- except (json.JSONDecodeError, OSError):
136
- pass
137
- if grading:
138
- break
139
-
140
- return {
141
- "id": run_id,
142
- "prompt": prompt,
143
- "eval_id": eval_id,
144
- "outputs": output_files,
145
- "grading": grading,
146
- }
147
-
148
-
149
- def embed_file(path: Path) -> dict:
150
- """Read a file and return an embedded representation."""
151
- ext = path.suffix.lower()
152
- mime = get_mime_type(path)
153
-
154
- if ext in TEXT_EXTENSIONS:
155
- try:
156
- content = path.read_text(errors="replace")
157
- except OSError:
158
- content = "(Error reading file)"
159
- return {
160
- "name": path.name,
161
- "type": "text",
162
- "content": content,
163
- }
164
- elif ext in IMAGE_EXTENSIONS:
165
- try:
166
- raw = path.read_bytes()
167
- b64 = base64.b64encode(raw).decode("ascii")
168
- except OSError:
169
- return {"name": path.name, "type": "error", "content": "(Error reading file)"}
170
- return {
171
- "name": path.name,
172
- "type": "image",
173
- "mime": mime,
174
- "data_uri": f"data:{mime};base64,{b64}",
175
- }
176
- elif ext == ".pdf":
177
- try:
178
- raw = path.read_bytes()
179
- b64 = base64.b64encode(raw).decode("ascii")
180
- except OSError:
181
- return {"name": path.name, "type": "error", "content": "(Error reading file)"}
182
- return {
183
- "name": path.name,
184
- "type": "pdf",
185
- "data_uri": f"data:{mime};base64,{b64}",
186
- }
187
- elif ext == ".xlsx":
188
- try:
189
- raw = path.read_bytes()
190
- b64 = base64.b64encode(raw).decode("ascii")
191
- except OSError:
192
- return {"name": path.name, "type": "error", "content": "(Error reading file)"}
193
- return {
194
- "name": path.name,
195
- "type": "xlsx",
196
- "data_b64": b64,
197
- }
198
- else:
199
- # Binary / unknown — base64 download link
200
- try:
201
- raw = path.read_bytes()
202
- b64 = base64.b64encode(raw).decode("ascii")
203
- except OSError:
204
- return {"name": path.name, "type": "error", "content": "(Error reading file)"}
205
- return {
206
- "name": path.name,
207
- "type": "binary",
208
- "mime": mime,
209
- "data_uri": f"data:{mime};base64,{b64}",
210
- }
211
-
212
-
213
- def load_previous_iteration(workspace: Path) -> dict[str, dict]:
214
- """Load previous iteration's feedback and outputs.
215
-
216
- Returns a map of run_id -> {"feedback": str, "outputs": list[dict]}.
217
- """
218
- result: dict[str, dict] = {}
219
-
220
- # Load feedback
221
- feedback_map: dict[str, str] = {}
222
- feedback_path = workspace / "feedback.json"
223
- if feedback_path.exists():
224
- try:
225
- data = json.loads(feedback_path.read_text())
226
- feedback_map = {
227
- r["run_id"]: r["feedback"]
228
- for r in data.get("reviews", [])
229
- if r.get("feedback", "").strip()
230
- }
231
- except (json.JSONDecodeError, OSError, KeyError):
232
- pass
233
-
234
- # Load runs (to get outputs)
235
- prev_runs = find_runs(workspace)
236
- for run in prev_runs:
237
- result[run["id"]] = {
238
- "feedback": feedback_map.get(run["id"], ""),
239
- "outputs": run.get("outputs", []),
240
- }
241
-
242
- # Also add feedback for run_ids that had feedback but no matching run
243
- for run_id, fb in feedback_map.items():
244
- if run_id not in result:
245
- result[run_id] = {"feedback": fb, "outputs": []}
246
-
247
- return result
248
-
249
-
250
- def generate_html(
251
- runs: list[dict],
252
- skill_name: str,
253
- previous: dict[str, dict] | None = None,
254
- benchmark: dict | None = None,
255
- ) -> str:
256
- """Generate the complete standalone HTML page with embedded data."""
257
- template_path = Path(__file__).parent / "viewer.html"
258
- template = template_path.read_text()
259
-
260
- # Build previous_feedback and previous_outputs maps for the template
261
- previous_feedback: dict[str, str] = {}
262
- previous_outputs: dict[str, list[dict]] = {}
263
- if previous:
264
- for run_id, data in previous.items():
265
- if data.get("feedback"):
266
- previous_feedback[run_id] = data["feedback"]
267
- if data.get("outputs"):
268
- previous_outputs[run_id] = data["outputs"]
269
-
270
- embedded = {
271
- "skill_name": skill_name,
272
- "runs": runs,
273
- "previous_feedback": previous_feedback,
274
- "previous_outputs": previous_outputs,
275
- }
276
- if benchmark:
277
- embedded["benchmark"] = benchmark
278
-
279
- data_json = json.dumps(embedded)
280
-
281
- return template.replace("/*__EMBEDDED_DATA__*/", f"const EMBEDDED_DATA = {data_json};")
282
-
283
-
284
- # ---------------------------------------------------------------------------
285
- # HTTP server (stdlib only, zero dependencies)
286
- # ---------------------------------------------------------------------------
287
-
288
- def _kill_port(port: int) -> None:
289
- """Kill any process listening on the given port."""
290
- try:
291
- result = subprocess.run(
292
- ["lsof", "-ti", f":{port}"],
293
- capture_output=True, text=True, timeout=5,
294
- )
295
- for pid_str in result.stdout.strip().split("\n"):
296
- if pid_str.strip():
297
- try:
298
- os.kill(int(pid_str.strip()), signal.SIGTERM)
299
- except (ProcessLookupError, ValueError):
300
- pass
301
- if result.stdout.strip():
302
- time.sleep(0.5)
303
- except subprocess.TimeoutExpired:
304
- pass
305
- except FileNotFoundError:
306
- print("Note: lsof not found, cannot check if port is in use", file=sys.stderr)
307
-
308
- class ReviewHandler(BaseHTTPRequestHandler):
309
- """Serves the review HTML and handles feedback saves.
310
-
311
- Regenerates the HTML on each page load so that refreshing the browser
312
- picks up new eval outputs without restarting the server.
313
- """
314
-
315
- def __init__(
316
- self,
317
- workspace: Path,
318
- skill_name: str,
319
- feedback_path: Path,
320
- previous: dict[str, dict],
321
- benchmark_path: Path | None,
322
- *args,
323
- **kwargs,
324
- ):
325
- self.workspace = workspace
326
- self.skill_name = skill_name
327
- self.feedback_path = feedback_path
328
- self.previous = previous
329
- self.benchmark_path = benchmark_path
330
- super().__init__(*args, **kwargs)
331
-
332
- def do_GET(self) -> None:
333
- if self.path == "/" or self.path == "/index.html":
334
- # Regenerate HTML on each request (re-scans workspace for new outputs)
335
- runs = find_runs(self.workspace)
336
- benchmark = None
337
- if self.benchmark_path and self.benchmark_path.exists():
338
- try:
339
- benchmark = json.loads(self.benchmark_path.read_text())
340
- except (json.JSONDecodeError, OSError):
341
- pass
342
- html = generate_html(runs, self.skill_name, self.previous, benchmark)
343
- content = html.encode("utf-8")
344
- self.send_response(200)
345
- self.send_header("Content-Type", "text/html; charset=utf-8")
346
- self.send_header("Content-Length", str(len(content)))
347
- self.end_headers()
348
- self.wfile.write(content)
349
- elif self.path == "/api/feedback":
350
- data = b"{}"
351
- if self.feedback_path.exists():
352
- data = self.feedback_path.read_bytes()
353
- self.send_response(200)
354
- self.send_header("Content-Type", "application/json")
355
- self.send_header("Content-Length", str(len(data)))
356
- self.end_headers()
357
- self.wfile.write(data)
358
- else:
359
- self.send_error(404)
360
-
361
- def do_POST(self) -> None:
362
- if self.path == "/api/feedback":
363
- length = int(self.headers.get("Content-Length", 0))
364
- body = self.rfile.read(length)
365
- try:
366
- data = json.loads(body)
367
- if not isinstance(data, dict) or "reviews" not in data:
368
- raise ValueError("Expected JSON object with 'reviews' key")
369
- self.feedback_path.write_text(json.dumps(data, indent=2) + "\n")
370
- resp = b'{"ok":true}'
371
- self.send_response(200)
372
- except (json.JSONDecodeError, OSError, ValueError) as e:
373
- resp = json.dumps({"error": str(e)}).encode()
374
- self.send_response(500)
375
- self.send_header("Content-Type", "application/json")
376
- self.send_header("Content-Length", str(len(resp)))
377
- self.end_headers()
378
- self.wfile.write(resp)
379
- else:
380
- self.send_error(404)
381
-
382
- def log_message(self, format: str, *args: object) -> None:
383
- # Suppress request logging to keep terminal clean
384
- pass
385
-
386
-
387
- def main() -> None:
388
- parser = argparse.ArgumentParser(description="Generate and serve eval review")
389
- parser.add_argument("workspace", type=Path, help="Path to workspace directory")
390
- parser.add_argument("--port", "-p", type=int, default=3117, help="Server port (default: 3117)")
391
- parser.add_argument("--skill-name", "-n", type=str, default=None, help="Skill name for header")
392
- parser.add_argument(
393
- "--previous-workspace", type=Path, default=None,
394
- help="Path to previous iteration's workspace (shows old outputs and feedback as context)",
395
- )
396
- parser.add_argument(
397
- "--benchmark", type=Path, default=None,
398
- help="Path to benchmark.json to show in the Benchmark tab",
399
- )
400
- parser.add_argument(
401
- "--static", "-s", type=Path, default=None,
402
- help="Write standalone HTML to this path instead of starting a server",
403
- )
404
- args = parser.parse_args()
405
-
406
- workspace = args.workspace.resolve()
407
- if not workspace.is_dir():
408
- print(f"Error: {workspace} is not a directory", file=sys.stderr)
409
- sys.exit(1)
410
-
411
- runs = find_runs(workspace)
412
- if not runs:
413
- print(f"No runs found in {workspace}", file=sys.stderr)
414
- sys.exit(1)
415
-
416
- skill_name = args.skill_name or workspace.name.replace("-workspace", "")
417
- feedback_path = workspace / "feedback.json"
418
-
419
- previous: dict[str, dict] = {}
420
- if args.previous_workspace:
421
- previous = load_previous_iteration(args.previous_workspace.resolve())
422
-
423
- benchmark_path = args.benchmark.resolve() if args.benchmark else None
424
- benchmark = None
425
- if benchmark_path and benchmark_path.exists():
426
- try:
427
- benchmark = json.loads(benchmark_path.read_text())
428
- except (json.JSONDecodeError, OSError):
429
- pass
430
-
431
- if args.static:
432
- html = generate_html(runs, skill_name, previous, benchmark)
433
- args.static.parent.mkdir(parents=True, exist_ok=True)
434
- args.static.write_text(html)
435
- print(f"\n Static viewer written to: {args.static}\n")
436
- sys.exit(0)
437
-
438
- # Kill any existing process on the target port
439
- port = args.port
440
- _kill_port(port)
441
- handler = partial(ReviewHandler, workspace, skill_name, feedback_path, previous, benchmark_path)
442
- try:
443
- server = HTTPServer(("127.0.0.1", port), handler)
444
- except OSError:
445
- # Port still in use after kill attempt — find a free one
446
- server = HTTPServer(("127.0.0.1", 0), handler)
447
- port = server.server_address[1]
448
-
449
- url = f"http://localhost:{port}"
450
- print(f"\n Eval Viewer")
451
- print(f" ─────────────────────────────────")
452
- print(f" URL: {url}")
453
- print(f" Workspace: {workspace}")
454
- print(f" Feedback: {feedback_path}")
455
- if previous:
456
- print(f" Previous: {args.previous_workspace} ({len(previous)} runs)")
457
- if benchmark_path:
458
- print(f" Benchmark: {benchmark_path}")
459
- print(f"\n Press Ctrl+C to stop.\n")
460
-
461
- webbrowser.open(url)
462
-
463
- try:
464
- server.serve_forever()
465
- except KeyboardInterrupt:
466
- print("\nStopped.")
467
- server.server_close()
468
-
469
-
470
- if __name__ == "__main__":
471
- main()