ai-engineering-init 1.7.0 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/.claude/hooks/skill-forced-eval.js +46 -62
  2. package/.claude/settings.json +10 -1
  3. package/.claude/skills/api-development/SKILL.md +179 -130
  4. package/.claude/skills/architecture-design/SKILL.md +102 -212
  5. package/.claude/skills/backend-annotations/SKILL.md +166 -220
  6. package/.claude/skills/bug-detective/SKILL.md +225 -186
  7. package/.claude/skills/code-patterns/SKILL.md +127 -244
  8. package/.claude/skills/collaborating-with-codex/SKILL.md +96 -113
  9. package/.claude/skills/crud-development/SKILL.md +226 -307
  10. package/.claude/skills/data-permission/SKILL.md +131 -202
  11. package/.claude/skills/database-ops/SKILL.md +158 -355
  12. package/.claude/skills/error-handler/SKILL.md +224 -285
  13. package/.claude/skills/file-oss-management/SKILL.md +174 -169
  14. package/.claude/skills/git-workflow/SKILL.md +123 -341
  15. package/.claude/skills/json-serialization/SKILL.md +121 -137
  16. package/.claude/skills/performance-doctor/SKILL.md +83 -89
  17. package/.claude/skills/redis-cache/SKILL.md +134 -185
  18. package/.claude/skills/scheduled-jobs/SKILL.md +187 -224
  19. package/.claude/skills/security-guard/SKILL.md +168 -276
  20. package/.claude/skills/sms-mail/SKILL.md +266 -228
  21. package/.claude/skills/social-login/SKILL.md +257 -195
  22. package/.claude/skills/tenant-management/SKILL.md +172 -188
  23. package/.claude/skills/utils-toolkit/SKILL.md +214 -222
  24. package/.claude/skills/websocket-sse/SKILL.md +251 -172
  25. package/.claude/skills/workflow-engine/SKILL.md +178 -250
  26. package/.codex/skills/api-development/SKILL.md +179 -130
  27. package/.codex/skills/architecture-design/SKILL.md +102 -212
  28. package/.codex/skills/backend-annotations/SKILL.md +166 -220
  29. package/.codex/skills/bug-detective/SKILL.md +225 -186
  30. package/.codex/skills/code-patterns/SKILL.md +127 -244
  31. package/.codex/skills/collaborating-with-codex/SKILL.md +96 -113
  32. package/.codex/skills/crud-development/SKILL.md +226 -307
  33. package/.codex/skills/data-permission/SKILL.md +131 -202
  34. package/.codex/skills/database-ops/SKILL.md +158 -355
  35. package/.codex/skills/error-handler/SKILL.md +224 -285
  36. package/.codex/skills/file-oss-management/SKILL.md +174 -169
  37. package/.codex/skills/git-workflow/SKILL.md +123 -341
  38. package/.codex/skills/json-serialization/SKILL.md +121 -137
  39. package/.codex/skills/performance-doctor/SKILL.md +83 -89
  40. package/.codex/skills/redis-cache/SKILL.md +134 -185
  41. package/.codex/skills/scheduled-jobs/SKILL.md +187 -224
  42. package/.codex/skills/security-guard/SKILL.md +168 -276
  43. package/.codex/skills/sms-mail/SKILL.md +266 -228
  44. package/.codex/skills/social-login/SKILL.md +257 -195
  45. package/.codex/skills/tenant-management/SKILL.md +172 -188
  46. package/.codex/skills/utils-toolkit/SKILL.md +214 -222
  47. package/.codex/skills/websocket-sse/SKILL.md +251 -172
  48. package/.codex/skills/workflow-engine/SKILL.md +178 -250
  49. package/.cursor/hooks/cursor-skill-eval.js +66 -6
  50. package/.cursor/skills/api-development/SKILL.md +179 -130
  51. package/.cursor/skills/architecture-design/SKILL.md +102 -212
  52. package/.cursor/skills/backend-annotations/SKILL.md +166 -220
  53. package/.cursor/skills/bug-detective/SKILL.md +225 -186
  54. package/.cursor/skills/code-patterns/SKILL.md +127 -244
  55. package/.cursor/skills/collaborating-with-codex/SKILL.md +96 -113
  56. package/.cursor/skills/crud-development/SKILL.md +226 -307
  57. package/.cursor/skills/data-permission/SKILL.md +131 -202
  58. package/.cursor/skills/database-ops/SKILL.md +158 -355
  59. package/.cursor/skills/error-handler/SKILL.md +224 -285
  60. package/.cursor/skills/file-oss-management/SKILL.md +174 -169
  61. package/.cursor/skills/git-workflow/SKILL.md +123 -341
  62. package/.cursor/skills/json-serialization/SKILL.md +121 -137
  63. package/.cursor/skills/performance-doctor/SKILL.md +83 -89
  64. package/.cursor/skills/redis-cache/SKILL.md +134 -185
  65. package/.cursor/skills/scheduled-jobs/SKILL.md +187 -224
  66. package/.cursor/skills/security-guard/SKILL.md +168 -276
  67. package/.cursor/skills/sms-mail/SKILL.md +266 -228
  68. package/.cursor/skills/social-login/SKILL.md +257 -195
  69. package/.cursor/skills/tenant-management/SKILL.md +172 -188
  70. package/.cursor/skills/utils-toolkit/SKILL.md +214 -222
  71. package/.cursor/skills/websocket-sse/SKILL.md +251 -172
  72. package/.cursor/skills/workflow-engine/SKILL.md +178 -250
  73. package/AGENTS.md +49 -540
  74. package/CLAUDE.md +73 -119
  75. package/README.md +37 -6
  76. package/bin/index.js +5 -1
  77. package/package.json +1 -1
  78. package/src/skills/api-development/SKILL.md +179 -130
  79. package/src/skills/architecture-design/SKILL.md +102 -212
  80. package/src/skills/backend-annotations/SKILL.md +166 -220
  81. package/src/skills/bug-detective/SKILL.md +225 -186
  82. package/src/skills/code-patterns/SKILL.md +127 -244
  83. package/src/skills/collaborating-with-codex/SKILL.md +96 -113
  84. package/src/skills/crud-development/SKILL.md +226 -307
  85. package/src/skills/data-permission/SKILL.md +131 -202
  86. package/src/skills/database-ops/SKILL.md +158 -355
  87. package/src/skills/error-handler/SKILL.md +224 -285
  88. package/src/skills/file-oss-management/SKILL.md +174 -169
  89. package/src/skills/git-workflow/SKILL.md +123 -341
  90. package/src/skills/json-serialization/SKILL.md +121 -137
  91. package/src/skills/performance-doctor/SKILL.md +83 -89
  92. package/src/skills/redis-cache/SKILL.md +134 -185
  93. package/src/skills/scheduled-jobs/SKILL.md +187 -224
  94. package/src/skills/security-guard/SKILL.md +168 -276
  95. package/src/skills/sms-mail/SKILL.md +266 -228
  96. package/src/skills/social-login/SKILL.md +257 -195
  97. package/src/skills/tenant-management/SKILL.md +172 -188
  98. package/src/skills/utils-toolkit/SKILL.md +214 -222
  99. package/src/skills/websocket-sse/SKILL.md +251 -172
  100. package/src/skills/workflow-engine/SKILL.md +178 -250
  101. package/.claude/skills/skill-creator/LICENSE.txt +0 -202
  102. package/.claude/skills/skill-creator/SKILL.md +0 -479
  103. package/.claude/skills/skill-creator/agents/analyzer.md +0 -274
  104. package/.claude/skills/skill-creator/agents/comparator.md +0 -202
  105. package/.claude/skills/skill-creator/agents/grader.md +0 -223
  106. package/.claude/skills/skill-creator/assets/eval_review.html +0 -146
  107. package/.claude/skills/skill-creator/eval-viewer/generate_review.py +0 -471
  108. package/.claude/skills/skill-creator/eval-viewer/viewer.html +0 -1325
  109. package/.claude/skills/skill-creator/references/schemas.md +0 -430
  110. package/.claude/skills/skill-creator/scripts/__init__.py +0 -0
  111. package/.claude/skills/skill-creator/scripts/aggregate_benchmark.py +0 -401
  112. package/.claude/skills/skill-creator/scripts/generate_report.py +0 -326
  113. package/.claude/skills/skill-creator/scripts/improve_description.py +0 -248
  114. package/.claude/skills/skill-creator/scripts/package_skill.py +0 -136
  115. package/.claude/skills/skill-creator/scripts/quick_validate.py +0 -103
  116. package/.claude/skills/skill-creator/scripts/run_eval.py +0 -310
  117. package/.claude/skills/skill-creator/scripts/run_loop.py +0 -332
  118. package/.claude/skills/skill-creator/scripts/utils.py +0 -47
@@ -1,326 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Generate an HTML report from run_loop.py output.
3
-
4
- Takes the JSON output from run_loop.py and generates a visual HTML report
5
- showing each description attempt with check/x for each test case.
6
- Distinguishes between train and test queries.
7
- """
8
-
9
- import argparse
10
- import html
11
- import json
12
- import sys
13
- from pathlib import Path
14
-
15
-
16
- def generate_html(data: dict, auto_refresh: bool = False, skill_name: str = "") -> str:
17
- """Generate HTML report from loop output data. If auto_refresh is True, adds a meta refresh tag."""
18
- history = data.get("history", [])
19
- holdout = data.get("holdout", 0)
20
- title_prefix = html.escape(skill_name + " \u2014 ") if skill_name else ""
21
-
22
- # Get all unique queries from train and test sets, with should_trigger info
23
- train_queries: list[dict] = []
24
- test_queries: list[dict] = []
25
- if history:
26
- for r in history[0].get("train_results", history[0].get("results", [])):
27
- train_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)})
28
- if history[0].get("test_results"):
29
- for r in history[0].get("test_results", []):
30
- test_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)})
31
-
32
- refresh_tag = ' <meta http-equiv="refresh" content="5">\n' if auto_refresh else ""
33
-
34
- html_parts = ["""<!DOCTYPE html>
35
- <html>
36
- <head>
37
- <meta charset="utf-8">
38
- """ + refresh_tag + """ <title>""" + title_prefix + """Skill Description Optimization</title>
39
- <link rel="preconnect" href="https://fonts.googleapis.com">
40
- <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
41
- <link href="https://fonts.googleapis.com/css2?family=Poppins:wght@500;600&family=Lora:wght@400;500&display=swap" rel="stylesheet">
42
- <style>
43
- body {
44
- font-family: 'Lora', Georgia, serif;
45
- max-width: 100%;
46
- margin: 0 auto;
47
- padding: 20px;
48
- background: #faf9f5;
49
- color: #141413;
50
- }
51
- h1 { font-family: 'Poppins', sans-serif; color: #141413; }
52
- .explainer {
53
- background: white;
54
- padding: 15px;
55
- border-radius: 6px;
56
- margin-bottom: 20px;
57
- border: 1px solid #e8e6dc;
58
- color: #b0aea5;
59
- font-size: 0.875rem;
60
- line-height: 1.6;
61
- }
62
- .summary {
63
- background: white;
64
- padding: 15px;
65
- border-radius: 6px;
66
- margin-bottom: 20px;
67
- border: 1px solid #e8e6dc;
68
- }
69
- .summary p { margin: 5px 0; }
70
- .best { color: #788c5d; font-weight: bold; }
71
- .table-container {
72
- overflow-x: auto;
73
- width: 100%;
74
- }
75
- table {
76
- border-collapse: collapse;
77
- background: white;
78
- border: 1px solid #e8e6dc;
79
- border-radius: 6px;
80
- font-size: 12px;
81
- min-width: 100%;
82
- }
83
- th, td {
84
- padding: 8px;
85
- text-align: left;
86
- border: 1px solid #e8e6dc;
87
- white-space: normal;
88
- word-wrap: break-word;
89
- }
90
- th {
91
- font-family: 'Poppins', sans-serif;
92
- background: #141413;
93
- color: #faf9f5;
94
- font-weight: 500;
95
- }
96
- th.test-col {
97
- background: #6a9bcc;
98
- }
99
- th.query-col { min-width: 200px; }
100
- td.description {
101
- font-family: monospace;
102
- font-size: 11px;
103
- word-wrap: break-word;
104
- max-width: 400px;
105
- }
106
- td.result {
107
- text-align: center;
108
- font-size: 16px;
109
- min-width: 40px;
110
- }
111
- td.test-result {
112
- background: #f0f6fc;
113
- }
114
- .pass { color: #788c5d; }
115
- .fail { color: #c44; }
116
- .rate {
117
- font-size: 9px;
118
- color: #b0aea5;
119
- display: block;
120
- }
121
- tr:hover { background: #faf9f5; }
122
- .score {
123
- display: inline-block;
124
- padding: 2px 6px;
125
- border-radius: 4px;
126
- font-weight: bold;
127
- font-size: 11px;
128
- }
129
- .score-good { background: #eef2e8; color: #788c5d; }
130
- .score-ok { background: #fef3c7; color: #d97706; }
131
- .score-bad { background: #fceaea; color: #c44; }
132
- .train-label { color: #b0aea5; font-size: 10px; }
133
- .test-label { color: #6a9bcc; font-size: 10px; font-weight: bold; }
134
- .best-row { background: #f5f8f2; }
135
- th.positive-col { border-bottom: 3px solid #788c5d; }
136
- th.negative-col { border-bottom: 3px solid #c44; }
137
- th.test-col.positive-col { border-bottom: 3px solid #788c5d; }
138
- th.test-col.negative-col { border-bottom: 3px solid #c44; }
139
- .legend { font-family: 'Poppins', sans-serif; display: flex; gap: 20px; margin-bottom: 10px; font-size: 13px; align-items: center; }
140
- .legend-item { display: flex; align-items: center; gap: 6px; }
141
- .legend-swatch { width: 16px; height: 16px; border-radius: 3px; display: inline-block; }
142
- .swatch-positive { background: #141413; border-bottom: 3px solid #788c5d; }
143
- .swatch-negative { background: #141413; border-bottom: 3px solid #c44; }
144
- .swatch-test { background: #6a9bcc; }
145
- .swatch-train { background: #141413; }
146
- </style>
147
- </head>
148
- <body>
149
- <h1>""" + title_prefix + """Skill Description Optimization</h1>
150
- <div class="explainer">
151
- <strong>Optimizing your skill's description.</strong> This page updates automatically as Claude tests different versions of your skill's description. Each row is an iteration — a new description attempt. The columns show test queries: green checkmarks mean the skill triggered correctly (or correctly didn't trigger), red crosses mean it got it wrong. The "Train" score shows performance on queries used to improve the description; the "Test" score shows performance on held-out queries the optimizer hasn't seen. When it's done, Claude will apply the best-performing description to your skill.
152
- </div>
153
- """]
154
-
155
- # Summary section
156
- best_test_score = data.get('best_test_score')
157
- best_train_score = data.get('best_train_score')
158
- html_parts.append(f"""
159
- <div class="summary">
160
- <p><strong>Original:</strong> {html.escape(data.get('original_description', 'N/A'))}</p>
161
- <p class="best"><strong>Best:</strong> {html.escape(data.get('best_description', 'N/A'))}</p>
162
- <p><strong>Best Score:</strong> {data.get('best_score', 'N/A')} {'(test)' if best_test_score else '(train)'}</p>
163
- <p><strong>Iterations:</strong> {data.get('iterations_run', 0)} | <strong>Train:</strong> {data.get('train_size', '?')} | <strong>Test:</strong> {data.get('test_size', '?')}</p>
164
- </div>
165
- """)
166
-
167
- # Legend
168
- html_parts.append("""
169
- <div class="legend">
170
- <span style="font-weight:600">Query columns:</span>
171
- <span class="legend-item"><span class="legend-swatch swatch-positive"></span> Should trigger</span>
172
- <span class="legend-item"><span class="legend-swatch swatch-negative"></span> Should NOT trigger</span>
173
- <span class="legend-item"><span class="legend-swatch swatch-train"></span> Train</span>
174
- <span class="legend-item"><span class="legend-swatch swatch-test"></span> Test</span>
175
- </div>
176
- """)
177
-
178
- # Table header
179
- html_parts.append("""
180
- <div class="table-container">
181
- <table>
182
- <thead>
183
- <tr>
184
- <th>Iter</th>
185
- <th>Train</th>
186
- <th>Test</th>
187
- <th class="query-col">Description</th>
188
- """)
189
-
190
- # Add column headers for train queries
191
- for qinfo in train_queries:
192
- polarity = "positive-col" if qinfo["should_trigger"] else "negative-col"
193
- html_parts.append(f' <th class="{polarity}">{html.escape(qinfo["query"])}</th>\n')
194
-
195
- # Add column headers for test queries (different color)
196
- for qinfo in test_queries:
197
- polarity = "positive-col" if qinfo["should_trigger"] else "negative-col"
198
- html_parts.append(f' <th class="test-col {polarity}">{html.escape(qinfo["query"])}</th>\n')
199
-
200
- html_parts.append(""" </tr>
201
- </thead>
202
- <tbody>
203
- """)
204
-
205
- # Find best iteration for highlighting
206
- if test_queries:
207
- best_iter = max(history, key=lambda h: h.get("test_passed") or 0).get("iteration")
208
- else:
209
- best_iter = max(history, key=lambda h: h.get("train_passed", h.get("passed", 0))).get("iteration")
210
-
211
- # Add rows for each iteration
212
- for h in history:
213
- iteration = h.get("iteration", "?")
214
- train_passed = h.get("train_passed", h.get("passed", 0))
215
- train_total = h.get("train_total", h.get("total", 0))
216
- test_passed = h.get("test_passed")
217
- test_total = h.get("test_total")
218
- description = h.get("description", "")
219
- train_results = h.get("train_results", h.get("results", []))
220
- test_results = h.get("test_results", [])
221
-
222
- # Create lookups for results by query
223
- train_by_query = {r["query"]: r for r in train_results}
224
- test_by_query = {r["query"]: r for r in test_results} if test_results else {}
225
-
226
- # Compute aggregate correct/total runs across all retries
227
- def aggregate_runs(results: list[dict]) -> tuple[int, int]:
228
- correct = 0
229
- total = 0
230
- for r in results:
231
- runs = r.get("runs", 0)
232
- triggers = r.get("triggers", 0)
233
- total += runs
234
- if r.get("should_trigger", True):
235
- correct += triggers
236
- else:
237
- correct += runs - triggers
238
- return correct, total
239
-
240
- train_correct, train_runs = aggregate_runs(train_results)
241
- test_correct, test_runs = aggregate_runs(test_results)
242
-
243
- # Determine score classes
244
- def score_class(correct: int, total: int) -> str:
245
- if total > 0:
246
- ratio = correct / total
247
- if ratio >= 0.8:
248
- return "score-good"
249
- elif ratio >= 0.5:
250
- return "score-ok"
251
- return "score-bad"
252
-
253
- train_class = score_class(train_correct, train_runs)
254
- test_class = score_class(test_correct, test_runs)
255
-
256
- row_class = "best-row" if iteration == best_iter else ""
257
-
258
- html_parts.append(f""" <tr class="{row_class}">
259
- <td>{iteration}</td>
260
- <td><span class="score {train_class}">{train_correct}/{train_runs}</span></td>
261
- <td><span class="score {test_class}">{test_correct}/{test_runs}</span></td>
262
- <td class="description">{html.escape(description)}</td>
263
- """)
264
-
265
- # Add result for each train query
266
- for qinfo in train_queries:
267
- r = train_by_query.get(qinfo["query"], {})
268
- did_pass = r.get("pass", False)
269
- triggers = r.get("triggers", 0)
270
- runs = r.get("runs", 0)
271
-
272
- icon = "✓" if did_pass else "✗"
273
- css_class = "pass" if did_pass else "fail"
274
-
275
- html_parts.append(f' <td class="result {css_class}">{icon}<span class="rate">{triggers}/{runs}</span></td>\n')
276
-
277
- # Add result for each test query (with different background)
278
- for qinfo in test_queries:
279
- r = test_by_query.get(qinfo["query"], {})
280
- did_pass = r.get("pass", False)
281
- triggers = r.get("triggers", 0)
282
- runs = r.get("runs", 0)
283
-
284
- icon = "✓" if did_pass else "✗"
285
- css_class = "pass" if did_pass else "fail"
286
-
287
- html_parts.append(f' <td class="result test-result {css_class}">{icon}<span class="rate">{triggers}/{runs}</span></td>\n')
288
-
289
- html_parts.append(" </tr>\n")
290
-
291
- html_parts.append(""" </tbody>
292
- </table>
293
- </div>
294
- """)
295
-
296
- html_parts.append("""
297
- </body>
298
- </html>
299
- """)
300
-
301
- return "".join(html_parts)
302
-
303
-
304
- def main():
305
- parser = argparse.ArgumentParser(description="Generate HTML report from run_loop output")
306
- parser.add_argument("input", help="Path to JSON output from run_loop.py (or - for stdin)")
307
- parser.add_argument("-o", "--output", default=None, help="Output HTML file (default: stdout)")
308
- parser.add_argument("--skill-name", default="", help="Skill name to include in the report title")
309
- args = parser.parse_args()
310
-
311
- if args.input == "-":
312
- data = json.load(sys.stdin)
313
- else:
314
- data = json.loads(Path(args.input).read_text())
315
-
316
- html_output = generate_html(data, skill_name=args.skill_name)
317
-
318
- if args.output:
319
- Path(args.output).write_text(html_output)
320
- print(f"Report written to {args.output}", file=sys.stderr)
321
- else:
322
- print(html_output)
323
-
324
-
325
- if __name__ == "__main__":
326
- main()
@@ -1,248 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Improve a skill description based on eval results.
3
-
4
- Takes eval results (from run_eval.py) and generates an improved description
5
- using Claude with extended thinking.
6
- """
7
-
8
- import argparse
9
- import json
10
- import re
11
- import sys
12
- from pathlib import Path
13
-
14
- import anthropic
15
-
16
- from scripts.utils import parse_skill_md
17
-
18
-
19
- def improve_description(
20
- client: anthropic.Anthropic,
21
- skill_name: str,
22
- skill_content: str,
23
- current_description: str,
24
- eval_results: dict,
25
- history: list[dict],
26
- model: str,
27
- test_results: dict | None = None,
28
- log_dir: Path | None = None,
29
- iteration: int | None = None,
30
- ) -> str:
31
- """Call Claude to improve the description based on eval results."""
32
- failed_triggers = [
33
- r for r in eval_results["results"]
34
- if r["should_trigger"] and not r["pass"]
35
- ]
36
- false_triggers = [
37
- r for r in eval_results["results"]
38
- if not r["should_trigger"] and not r["pass"]
39
- ]
40
-
41
- # Build scores summary
42
- train_score = f"{eval_results['summary']['passed']}/{eval_results['summary']['total']}"
43
- if test_results:
44
- test_score = f"{test_results['summary']['passed']}/{test_results['summary']['total']}"
45
- scores_summary = f"Train: {train_score}, Test: {test_score}"
46
- else:
47
- scores_summary = f"Train: {train_score}"
48
-
49
- prompt = f"""You are optimizing a skill description for a Claude Code skill called "{skill_name}". A "skill" is sort of like a prompt, but with progressive disclosure -- there's a title and description that Claude sees when deciding whether to use the skill, and then if it does use the skill, it reads the .md file which has lots more details and potentially links to other resources in the skill folder like helper files and scripts and additional documentation or examples.
50
-
51
- The description appears in Claude's "available_skills" list. When a user sends a query, Claude decides whether to invoke the skill based solely on the title and on this description. Your goal is to write a description that triggers for relevant queries, and doesn't trigger for irrelevant ones.
52
-
53
- Here's the current description:
54
- <current_description>
55
- "{current_description}"
56
- </current_description>
57
-
58
- Current scores ({scores_summary}):
59
- <scores_summary>
60
- """
61
- if failed_triggers:
62
- prompt += "FAILED TO TRIGGER (should have triggered but didn't):\n"
63
- for r in failed_triggers:
64
- prompt += f' - "{r["query"]}" (triggered {r["triggers"]}/{r["runs"]} times)\n'
65
- prompt += "\n"
66
-
67
- if false_triggers:
68
- prompt += "FALSE TRIGGERS (triggered but shouldn't have):\n"
69
- for r in false_triggers:
70
- prompt += f' - "{r["query"]}" (triggered {r["triggers"]}/{r["runs"]} times)\n'
71
- prompt += "\n"
72
-
73
- if history:
74
- prompt += "PREVIOUS ATTEMPTS (do NOT repeat these — try something structurally different):\n\n"
75
- for h in history:
76
- train_s = f"{h.get('train_passed', h.get('passed', 0))}/{h.get('train_total', h.get('total', 0))}"
77
- test_s = f"{h.get('test_passed', '?')}/{h.get('test_total', '?')}" if h.get('test_passed') is not None else None
78
- score_str = f"train={train_s}" + (f", test={test_s}" if test_s else "")
79
- prompt += f'<attempt {score_str}>\n'
80
- prompt += f'Description: "{h["description"]}"\n'
81
- if "results" in h:
82
- prompt += "Train results:\n"
83
- for r in h["results"]:
84
- status = "PASS" if r["pass"] else "FAIL"
85
- prompt += f' [{status}] "{r["query"][:80]}" (triggered {r["triggers"]}/{r["runs"]})\n'
86
- if h.get("note"):
87
- prompt += f'Note: {h["note"]}\n'
88
- prompt += "</attempt>\n\n"
89
-
90
- prompt += f"""</scores_summary>
91
-
92
- Skill content (for context on what the skill does):
93
- <skill_content>
94
- {skill_content}
95
- </skill_content>
96
-
97
- Based on the failures, write a new and improved description that is more likely to trigger correctly. When I say "based on the failures", it's a bit of a tricky line to walk because we don't want to overfit to the specific cases you're seeing. So what I DON'T want you to do is produce an ever-expanding list of specific queries that this skill should or shouldn't trigger for. Instead, try to generalize from the failures to broader categories of user intent and situations where this skill would be useful or not useful. The reason for this is twofold:
98
-
99
- 1. Avoid overfitting
100
- 2. The list might get loooong and it's injected into ALL queries and there might be a lot of skills, so we don't want to blow too much space on any given description.
101
-
102
- Concretely, your description should not be more than about 100-200 words, even if that comes at the cost of accuracy.
103
-
104
- Here are some tips that we've found to work well in writing these descriptions:
105
- - The skill should be phrased in the imperative -- "Use this skill for" rather than "this skill does"
106
- - The skill description should focus on the user's intent, what they are trying to achieve, vs. the implementation details of how the skill works.
107
- - The description competes with other skills for Claude's attention — make it distinctive and immediately recognizable.
108
- - If you're getting lots of failures after repeated attempts, change things up. Try different sentence structures or wordings.
109
-
110
- I'd encourage you to be creative and mix up the style in different iterations since you'll have multiple opportunities to try different approaches and we'll just grab the highest-scoring one at the end.
111
-
112
- Please respond with only the new description text in <new_description> tags, nothing else."""
113
-
114
- response = client.messages.create(
115
- model=model,
116
- max_tokens=16000,
117
- thinking={
118
- "type": "enabled",
119
- "budget_tokens": 10000,
120
- },
121
- messages=[{"role": "user", "content": prompt}],
122
- )
123
-
124
- # Extract thinking and text from response
125
- thinking_text = ""
126
- text = ""
127
- for block in response.content:
128
- if block.type == "thinking":
129
- thinking_text = block.thinking
130
- elif block.type == "text":
131
- text = block.text
132
-
133
- # Parse out the <new_description> tags
134
- match = re.search(r"<new_description>(.*?)</new_description>", text, re.DOTALL)
135
- description = match.group(1).strip().strip('"') if match else text.strip().strip('"')
136
-
137
- # Log the transcript
138
- transcript: dict = {
139
- "iteration": iteration,
140
- "prompt": prompt,
141
- "thinking": thinking_text,
142
- "response": text,
143
- "parsed_description": description,
144
- "char_count": len(description),
145
- "over_limit": len(description) > 1024,
146
- }
147
-
148
- # If over 1024 chars, ask the model to shorten it
149
- if len(description) > 1024:
150
- shorten_prompt = f"Your description is {len(description)} characters, which exceeds the hard 1024 character limit. Please rewrite it to be under 1024 characters while preserving the most important trigger words and intent coverage. Respond with only the new description in <new_description> tags."
151
- shorten_response = client.messages.create(
152
- model=model,
153
- max_tokens=16000,
154
- thinking={
155
- "type": "enabled",
156
- "budget_tokens": 10000,
157
- },
158
- messages=[
159
- {"role": "user", "content": prompt},
160
- {"role": "assistant", "content": text},
161
- {"role": "user", "content": shorten_prompt},
162
- ],
163
- )
164
-
165
- shorten_thinking = ""
166
- shorten_text = ""
167
- for block in shorten_response.content:
168
- if block.type == "thinking":
169
- shorten_thinking = block.thinking
170
- elif block.type == "text":
171
- shorten_text = block.text
172
-
173
- match = re.search(r"<new_description>(.*?)</new_description>", shorten_text, re.DOTALL)
174
- shortened = match.group(1).strip().strip('"') if match else shorten_text.strip().strip('"')
175
-
176
- transcript["rewrite_prompt"] = shorten_prompt
177
- transcript["rewrite_thinking"] = shorten_thinking
178
- transcript["rewrite_response"] = shorten_text
179
- transcript["rewrite_description"] = shortened
180
- transcript["rewrite_char_count"] = len(shortened)
181
- description = shortened
182
-
183
- transcript["final_description"] = description
184
-
185
- if log_dir:
186
- log_dir.mkdir(parents=True, exist_ok=True)
187
- log_file = log_dir / f"improve_iter_{iteration or 'unknown'}.json"
188
- log_file.write_text(json.dumps(transcript, indent=2))
189
-
190
- return description
191
-
192
-
193
- def main():
194
- parser = argparse.ArgumentParser(description="Improve a skill description based on eval results")
195
- parser.add_argument("--eval-results", required=True, help="Path to eval results JSON (from run_eval.py)")
196
- parser.add_argument("--skill-path", required=True, help="Path to skill directory")
197
- parser.add_argument("--history", default=None, help="Path to history JSON (previous attempts)")
198
- parser.add_argument("--model", required=True, help="Model for improvement")
199
- parser.add_argument("--verbose", action="store_true", help="Print thinking to stderr")
200
- args = parser.parse_args()
201
-
202
- skill_path = Path(args.skill_path)
203
- if not (skill_path / "SKILL.md").exists():
204
- print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr)
205
- sys.exit(1)
206
-
207
- eval_results = json.loads(Path(args.eval_results).read_text())
208
- history = []
209
- if args.history:
210
- history = json.loads(Path(args.history).read_text())
211
-
212
- name, _, content = parse_skill_md(skill_path)
213
- current_description = eval_results["description"]
214
-
215
- if args.verbose:
216
- print(f"Current: {current_description}", file=sys.stderr)
217
- print(f"Score: {eval_results['summary']['passed']}/{eval_results['summary']['total']}", file=sys.stderr)
218
-
219
- client = anthropic.Anthropic()
220
- new_description = improve_description(
221
- client=client,
222
- skill_name=name,
223
- skill_content=content,
224
- current_description=current_description,
225
- eval_results=eval_results,
226
- history=history,
227
- model=args.model,
228
- )
229
-
230
- if args.verbose:
231
- print(f"Improved: {new_description}", file=sys.stderr)
232
-
233
- # Output as JSON with both the new description and updated history
234
- output = {
235
- "description": new_description,
236
- "history": history + [{
237
- "description": current_description,
238
- "passed": eval_results["summary"]["passed"],
239
- "failed": eval_results["summary"]["failed"],
240
- "total": eval_results["summary"]["total"],
241
- "results": eval_results["results"],
242
- }],
243
- }
244
- print(json.dumps(output, indent=2))
245
-
246
-
247
- if __name__ == "__main__":
248
- main()