ai-engineering-init 1.7.0 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/.claude/agents/bug-analyzer.md +103 -0
  2. package/.claude/agents/code-reviewer.md +115 -5
  3. package/.claude/agents/image-reader.md +154 -0
  4. package/.claude/agents/loki-runner.md +80 -0
  5. package/.claude/agents/mysql-runner.md +81 -0
  6. package/.claude/agents/requirements-analyzer.md +162 -0
  7. package/.claude/agents/task-fetcher.md +75 -0
  8. package/.claude/commands/dev.md +29 -0
  9. package/.claude/commands/next.md +31 -1
  10. package/.claude/commands/progress.md +23 -1
  11. package/.claude/hooks/skill-forced-eval.js +46 -62
  12. package/.claude/settings.json +10 -1
  13. package/.claude/skills/api-development/SKILL.md +179 -130
  14. package/.claude/skills/architecture-design/SKILL.md +102 -212
  15. package/.claude/skills/backend-annotations/SKILL.md +166 -220
  16. package/.claude/skills/bug-detective/SKILL.md +225 -186
  17. package/.claude/skills/code-patterns/SKILL.md +127 -244
  18. package/.claude/skills/collaborating-with-codex/SKILL.md +96 -113
  19. package/.claude/skills/crud-development/SKILL.md +226 -307
  20. package/.claude/skills/data-permission/SKILL.md +131 -202
  21. package/.claude/skills/database-ops/SKILL.md +158 -355
  22. package/.claude/skills/error-handler/SKILL.md +224 -285
  23. package/.claude/skills/file-oss-management/SKILL.md +174 -169
  24. package/.claude/skills/git-workflow/SKILL.md +123 -341
  25. package/.claude/skills/json-serialization/SKILL.md +121 -137
  26. package/.claude/skills/performance-doctor/SKILL.md +83 -89
  27. package/.claude/skills/redis-cache/SKILL.md +134 -185
  28. package/.claude/skills/scheduled-jobs/SKILL.md +187 -224
  29. package/.claude/skills/security-guard/SKILL.md +168 -276
  30. package/.claude/skills/sms-mail/SKILL.md +266 -228
  31. package/.claude/skills/social-login/SKILL.md +257 -195
  32. package/.claude/skills/tenant-management/SKILL.md +172 -188
  33. package/.claude/skills/utils-toolkit/SKILL.md +214 -222
  34. package/.claude/skills/websocket-sse/SKILL.md +251 -172
  35. package/.claude/skills/workflow-engine/SKILL.md +178 -250
  36. package/.codex/skills/api-development/SKILL.md +179 -130
  37. package/.codex/skills/architecture-design/SKILL.md +102 -212
  38. package/.codex/skills/backend-annotations/SKILL.md +166 -220
  39. package/.codex/skills/bug-detective/SKILL.md +225 -186
  40. package/.codex/skills/code-patterns/SKILL.md +127 -244
  41. package/.codex/skills/collaborating-with-codex/SKILL.md +96 -113
  42. package/.codex/skills/crud-development/SKILL.md +226 -307
  43. package/.codex/skills/data-permission/SKILL.md +131 -202
  44. package/.codex/skills/database-ops/SKILL.md +158 -355
  45. package/.codex/skills/dev/SKILL.md +476 -131
  46. package/.codex/skills/error-handler/SKILL.md +224 -285
  47. package/.codex/skills/file-oss-management/SKILL.md +174 -169
  48. package/.codex/skills/git-workflow/SKILL.md +123 -341
  49. package/.codex/skills/json-serialization/SKILL.md +121 -137
  50. package/.codex/skills/next/SKILL.md +186 -42
  51. package/.codex/skills/performance-doctor/SKILL.md +83 -89
  52. package/.codex/skills/progress/SKILL.md +147 -76
  53. package/.codex/skills/redis-cache/SKILL.md +134 -185
  54. package/.codex/skills/scheduled-jobs/SKILL.md +187 -224
  55. package/.codex/skills/security-guard/SKILL.md +168 -276
  56. package/.codex/skills/sms-mail/SKILL.md +266 -228
  57. package/.codex/skills/social-login/SKILL.md +257 -195
  58. package/.codex/skills/tenant-management/SKILL.md +172 -188
  59. package/.codex/skills/utils-toolkit/SKILL.md +214 -222
  60. package/.codex/skills/websocket-sse/SKILL.md +251 -172
  61. package/.codex/skills/workflow-engine/SKILL.md +178 -250
  62. package/.cursor/agents/bug-analyzer.md +102 -0
  63. package/.cursor/agents/code-reviewer.md +80 -97
  64. package/.cursor/agents/image-reader.md +154 -0
  65. package/.cursor/agents/loki-runner.md +80 -0
  66. package/.cursor/agents/mysql-runner.md +81 -0
  67. package/.cursor/agents/project-manager.md +1 -1
  68. package/.cursor/agents/requirements-analyzer.md +141 -0
  69. package/.cursor/agents/task-fetcher.md +75 -0
  70. package/.cursor/hooks/cursor-skill-eval.js +66 -6
  71. package/.cursor/skills/api-development/SKILL.md +179 -130
  72. package/.cursor/skills/architecture-design/SKILL.md +102 -212
  73. package/.cursor/skills/backend-annotations/SKILL.md +166 -220
  74. package/.cursor/skills/bug-detective/SKILL.md +225 -186
  75. package/.cursor/skills/code-patterns/SKILL.md +127 -244
  76. package/.cursor/skills/collaborating-with-codex/SKILL.md +96 -113
  77. package/.cursor/skills/crud-development/SKILL.md +226 -307
  78. package/.cursor/skills/data-permission/SKILL.md +131 -202
  79. package/.cursor/skills/database-ops/SKILL.md +158 -355
  80. package/.cursor/skills/error-handler/SKILL.md +224 -285
  81. package/.cursor/skills/file-oss-management/SKILL.md +174 -169
  82. package/.cursor/skills/git-workflow/SKILL.md +123 -341
  83. package/.cursor/skills/json-serialization/SKILL.md +121 -137
  84. package/.cursor/skills/performance-doctor/SKILL.md +83 -89
  85. package/.cursor/skills/redis-cache/SKILL.md +134 -185
  86. package/.cursor/skills/scheduled-jobs/SKILL.md +187 -224
  87. package/.cursor/skills/security-guard/SKILL.md +168 -276
  88. package/.cursor/skills/sms-mail/SKILL.md +266 -228
  89. package/.cursor/skills/social-login/SKILL.md +257 -195
  90. package/.cursor/skills/tenant-management/SKILL.md +172 -188
  91. package/.cursor/skills/utils-toolkit/SKILL.md +214 -222
  92. package/.cursor/skills/websocket-sse/SKILL.md +251 -172
  93. package/.cursor/skills/workflow-engine/SKILL.md +178 -250
  94. package/AGENTS.md +117 -540
  95. package/CLAUDE.md +105 -117
  96. package/README.md +37 -6
  97. package/bin/index.js +5 -1
  98. package/package.json +1 -1
  99. package/src/skills/api-development/SKILL.md +179 -130
  100. package/src/skills/architecture-design/SKILL.md +102 -212
  101. package/src/skills/backend-annotations/SKILL.md +166 -220
  102. package/src/skills/bug-detective/SKILL.md +225 -186
  103. package/src/skills/code-patterns/SKILL.md +127 -244
  104. package/src/skills/collaborating-with-codex/SKILL.md +96 -113
  105. package/src/skills/crud-development/SKILL.md +226 -307
  106. package/src/skills/data-permission/SKILL.md +131 -202
  107. package/src/skills/database-ops/SKILL.md +158 -355
  108. package/src/skills/error-handler/SKILL.md +224 -285
  109. package/src/skills/file-oss-management/SKILL.md +174 -169
  110. package/src/skills/git-workflow/SKILL.md +123 -341
  111. package/src/skills/json-serialization/SKILL.md +121 -137
  112. package/src/skills/performance-doctor/SKILL.md +83 -89
  113. package/src/skills/redis-cache/SKILL.md +134 -185
  114. package/src/skills/scheduled-jobs/SKILL.md +187 -224
  115. package/src/skills/security-guard/SKILL.md +168 -276
  116. package/src/skills/sms-mail/SKILL.md +266 -228
  117. package/src/skills/social-login/SKILL.md +257 -195
  118. package/src/skills/tenant-management/SKILL.md +172 -188
  119. package/src/skills/utils-toolkit/SKILL.md +214 -222
  120. package/src/skills/websocket-sse/SKILL.md +251 -172
  121. package/src/skills/workflow-engine/SKILL.md +178 -250
  122. package/.claude/skills/skill-creator/LICENSE.txt +0 -202
  123. package/.claude/skills/skill-creator/SKILL.md +0 -479
  124. package/.claude/skills/skill-creator/agents/analyzer.md +0 -274
  125. package/.claude/skills/skill-creator/agents/comparator.md +0 -202
  126. package/.claude/skills/skill-creator/agents/grader.md +0 -223
  127. package/.claude/skills/skill-creator/assets/eval_review.html +0 -146
  128. package/.claude/skills/skill-creator/eval-viewer/generate_review.py +0 -471
  129. package/.claude/skills/skill-creator/eval-viewer/viewer.html +0 -1325
  130. package/.claude/skills/skill-creator/references/schemas.md +0 -430
  131. package/.claude/skills/skill-creator/scripts/__init__.py +0 -0
  132. package/.claude/skills/skill-creator/scripts/aggregate_benchmark.py +0 -401
  133. package/.claude/skills/skill-creator/scripts/generate_report.py +0 -326
  134. package/.claude/skills/skill-creator/scripts/improve_description.py +0 -248
  135. package/.claude/skills/skill-creator/scripts/package_skill.py +0 -136
  136. package/.claude/skills/skill-creator/scripts/quick_validate.py +0 -103
  137. package/.claude/skills/skill-creator/scripts/run_eval.py +0 -310
  138. package/.claude/skills/skill-creator/scripts/run_loop.py +0 -332
  139. package/.claude/skills/skill-creator/scripts/utils.py +0 -47
package/.claude/skills/skill-creator/scripts/generate_report.py DELETED
@@ -1,326 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Generate an HTML report from run_loop.py output.
3
-
4
- Takes the JSON output from run_loop.py and generates a visual HTML report
5
- showing each description attempt with check/x for each test case.
6
- Distinguishes between train and test queries.
7
- """
8
-
9
- import argparse
10
- import html
11
- import json
12
- import sys
13
- from pathlib import Path
14
-
15
-
16
- def generate_html(data: dict, auto_refresh: bool = False, skill_name: str = "") -> str:
17
- """Generate HTML report from loop output data. If auto_refresh is True, adds a meta refresh tag."""
18
- history = data.get("history", [])
19
- holdout = data.get("holdout", 0)
20
- title_prefix = html.escape(skill_name + " \u2014 ") if skill_name else ""
21
-
22
- # Get all unique queries from train and test sets, with should_trigger info
23
- train_queries: list[dict] = []
24
- test_queries: list[dict] = []
25
- if history:
26
- for r in history[0].get("train_results", history[0].get("results", [])):
27
- train_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)})
28
- if history[0].get("test_results"):
29
- for r in history[0].get("test_results", []):
30
- test_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)})
31
-
32
- refresh_tag = ' <meta http-equiv="refresh" content="5">\n' if auto_refresh else ""
33
-
34
- html_parts = ["""<!DOCTYPE html>
35
- <html>
36
- <head>
37
- <meta charset="utf-8">
38
- """ + refresh_tag + """ <title>""" + title_prefix + """Skill Description Optimization</title>
39
- <link rel="preconnect" href="https://fonts.googleapis.com">
40
- <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
41
- <link href="https://fonts.googleapis.com/css2?family=Poppins:wght@500;600&family=Lora:wght@400;500&display=swap" rel="stylesheet">
42
- <style>
43
- body {
44
- font-family: 'Lora', Georgia, serif;
45
- max-width: 100%;
46
- margin: 0 auto;
47
- padding: 20px;
48
- background: #faf9f5;
49
- color: #141413;
50
- }
51
- h1 { font-family: 'Poppins', sans-serif; color: #141413; }
52
- .explainer {
53
- background: white;
54
- padding: 15px;
55
- border-radius: 6px;
56
- margin-bottom: 20px;
57
- border: 1px solid #e8e6dc;
58
- color: #b0aea5;
59
- font-size: 0.875rem;
60
- line-height: 1.6;
61
- }
62
- .summary {
63
- background: white;
64
- padding: 15px;
65
- border-radius: 6px;
66
- margin-bottom: 20px;
67
- border: 1px solid #e8e6dc;
68
- }
69
- .summary p { margin: 5px 0; }
70
- .best { color: #788c5d; font-weight: bold; }
71
- .table-container {
72
- overflow-x: auto;
73
- width: 100%;
74
- }
75
- table {
76
- border-collapse: collapse;
77
- background: white;
78
- border: 1px solid #e8e6dc;
79
- border-radius: 6px;
80
- font-size: 12px;
81
- min-width: 100%;
82
- }
83
- th, td {
84
- padding: 8px;
85
- text-align: left;
86
- border: 1px solid #e8e6dc;
87
- white-space: normal;
88
- word-wrap: break-word;
89
- }
90
- th {
91
- font-family: 'Poppins', sans-serif;
92
- background: #141413;
93
- color: #faf9f5;
94
- font-weight: 500;
95
- }
96
- th.test-col {
97
- background: #6a9bcc;
98
- }
99
- th.query-col { min-width: 200px; }
100
- td.description {
101
- font-family: monospace;
102
- font-size: 11px;
103
- word-wrap: break-word;
104
- max-width: 400px;
105
- }
106
- td.result {
107
- text-align: center;
108
- font-size: 16px;
109
- min-width: 40px;
110
- }
111
- td.test-result {
112
- background: #f0f6fc;
113
- }
114
- .pass { color: #788c5d; }
115
- .fail { color: #c44; }
116
- .rate {
117
- font-size: 9px;
118
- color: #b0aea5;
119
- display: block;
120
- }
121
- tr:hover { background: #faf9f5; }
122
- .score {
123
- display: inline-block;
124
- padding: 2px 6px;
125
- border-radius: 4px;
126
- font-weight: bold;
127
- font-size: 11px;
128
- }
129
- .score-good { background: #eef2e8; color: #788c5d; }
130
- .score-ok { background: #fef3c7; color: #d97706; }
131
- .score-bad { background: #fceaea; color: #c44; }
132
- .train-label { color: #b0aea5; font-size: 10px; }
133
- .test-label { color: #6a9bcc; font-size: 10px; font-weight: bold; }
134
- .best-row { background: #f5f8f2; }
135
- th.positive-col { border-bottom: 3px solid #788c5d; }
136
- th.negative-col { border-bottom: 3px solid #c44; }
137
- th.test-col.positive-col { border-bottom: 3px solid #788c5d; }
138
- th.test-col.negative-col { border-bottom: 3px solid #c44; }
139
- .legend { font-family: 'Poppins', sans-serif; display: flex; gap: 20px; margin-bottom: 10px; font-size: 13px; align-items: center; }
140
- .legend-item { display: flex; align-items: center; gap: 6px; }
141
- .legend-swatch { width: 16px; height: 16px; border-radius: 3px; display: inline-block; }
142
- .swatch-positive { background: #141413; border-bottom: 3px solid #788c5d; }
143
- .swatch-negative { background: #141413; border-bottom: 3px solid #c44; }
144
- .swatch-test { background: #6a9bcc; }
145
- .swatch-train { background: #141413; }
146
- </style>
147
- </head>
148
- <body>
149
- <h1>""" + title_prefix + """Skill Description Optimization</h1>
150
- <div class="explainer">
151
- <strong>Optimizing your skill's description.</strong> This page updates automatically as Claude tests different versions of your skill's description. Each row is an iteration — a new description attempt. The columns show test queries: green checkmarks mean the skill triggered correctly (or correctly didn't trigger), red crosses mean it got it wrong. The "Train" score shows performance on queries used to improve the description; the "Test" score shows performance on held-out queries the optimizer hasn't seen. When it's done, Claude will apply the best-performing description to your skill.
152
- </div>
153
- """]
154
-
155
- # Summary section
156
- best_test_score = data.get('best_test_score')
157
- best_train_score = data.get('best_train_score')
158
- html_parts.append(f"""
159
- <div class="summary">
160
- <p><strong>Original:</strong> {html.escape(data.get('original_description', 'N/A'))}</p>
161
- <p class="best"><strong>Best:</strong> {html.escape(data.get('best_description', 'N/A'))}</p>
162
- <p><strong>Best Score:</strong> {data.get('best_score', 'N/A')} {'(test)' if best_test_score else '(train)'}</p>
163
- <p><strong>Iterations:</strong> {data.get('iterations_run', 0)} | <strong>Train:</strong> {data.get('train_size', '?')} | <strong>Test:</strong> {data.get('test_size', '?')}</p>
164
- </div>
165
- """)
166
-
167
- # Legend
168
- html_parts.append("""
169
- <div class="legend">
170
- <span style="font-weight:600">Query columns:</span>
171
- <span class="legend-item"><span class="legend-swatch swatch-positive"></span> Should trigger</span>
172
- <span class="legend-item"><span class="legend-swatch swatch-negative"></span> Should NOT trigger</span>
173
- <span class="legend-item"><span class="legend-swatch swatch-train"></span> Train</span>
174
- <span class="legend-item"><span class="legend-swatch swatch-test"></span> Test</span>
175
- </div>
176
- """)
177
-
178
- # Table header
179
- html_parts.append("""
180
- <div class="table-container">
181
- <table>
182
- <thead>
183
- <tr>
184
- <th>Iter</th>
185
- <th>Train</th>
186
- <th>Test</th>
187
- <th class="query-col">Description</th>
188
- """)
189
-
190
- # Add column headers for train queries
191
- for qinfo in train_queries:
192
- polarity = "positive-col" if qinfo["should_trigger"] else "negative-col"
193
- html_parts.append(f' <th class="{polarity}">{html.escape(qinfo["query"])}</th>\n')
194
-
195
- # Add column headers for test queries (different color)
196
- for qinfo in test_queries:
197
- polarity = "positive-col" if qinfo["should_trigger"] else "negative-col"
198
- html_parts.append(f' <th class="test-col {polarity}">{html.escape(qinfo["query"])}</th>\n')
199
-
200
- html_parts.append(""" </tr>
201
- </thead>
202
- <tbody>
203
- """)
204
-
205
- # Find best iteration for highlighting
206
- if test_queries:
207
- best_iter = max(history, key=lambda h: h.get("test_passed") or 0).get("iteration")
208
- else:
209
- best_iter = max(history, key=lambda h: h.get("train_passed", h.get("passed", 0))).get("iteration")
210
-
211
- # Add rows for each iteration
212
- for h in history:
213
- iteration = h.get("iteration", "?")
214
- train_passed = h.get("train_passed", h.get("passed", 0))
215
- train_total = h.get("train_total", h.get("total", 0))
216
- test_passed = h.get("test_passed")
217
- test_total = h.get("test_total")
218
- description = h.get("description", "")
219
- train_results = h.get("train_results", h.get("results", []))
220
- test_results = h.get("test_results", [])
221
-
222
- # Create lookups for results by query
223
- train_by_query = {r["query"]: r for r in train_results}
224
- test_by_query = {r["query"]: r for r in test_results} if test_results else {}
225
-
226
- # Compute aggregate correct/total runs across all retries
227
- def aggregate_runs(results: list[dict]) -> tuple[int, int]:
228
- correct = 0
229
- total = 0
230
- for r in results:
231
- runs = r.get("runs", 0)
232
- triggers = r.get("triggers", 0)
233
- total += runs
234
- if r.get("should_trigger", True):
235
- correct += triggers
236
- else:
237
- correct += runs - triggers
238
- return correct, total
239
-
240
- train_correct, train_runs = aggregate_runs(train_results)
241
- test_correct, test_runs = aggregate_runs(test_results)
242
-
243
- # Determine score classes
244
- def score_class(correct: int, total: int) -> str:
245
- if total > 0:
246
- ratio = correct / total
247
- if ratio >= 0.8:
248
- return "score-good"
249
- elif ratio >= 0.5:
250
- return "score-ok"
251
- return "score-bad"
252
-
253
- train_class = score_class(train_correct, train_runs)
254
- test_class = score_class(test_correct, test_runs)
255
-
256
- row_class = "best-row" if iteration == best_iter else ""
257
-
258
- html_parts.append(f""" <tr class="{row_class}">
259
- <td>{iteration}</td>
260
- <td><span class="score {train_class}">{train_correct}/{train_runs}</span></td>
261
- <td><span class="score {test_class}">{test_correct}/{test_runs}</span></td>
262
- <td class="description">{html.escape(description)}</td>
263
- """)
264
-
265
- # Add result for each train query
266
- for qinfo in train_queries:
267
- r = train_by_query.get(qinfo["query"], {})
268
- did_pass = r.get("pass", False)
269
- triggers = r.get("triggers", 0)
270
- runs = r.get("runs", 0)
271
-
272
- icon = "✓" if did_pass else "✗"
273
- css_class = "pass" if did_pass else "fail"
274
-
275
- html_parts.append(f' <td class="result {css_class}">{icon}<span class="rate">{triggers}/{runs}</span></td>\n')
276
-
277
- # Add result for each test query (with different background)
278
- for qinfo in test_queries:
279
- r = test_by_query.get(qinfo["query"], {})
280
- did_pass = r.get("pass", False)
281
- triggers = r.get("triggers", 0)
282
- runs = r.get("runs", 0)
283
-
284
- icon = "✓" if did_pass else "✗"
285
- css_class = "pass" if did_pass else "fail"
286
-
287
- html_parts.append(f' <td class="result test-result {css_class}">{icon}<span class="rate">{triggers}/{runs}</span></td>\n')
288
-
289
- html_parts.append(" </tr>\n")
290
-
291
- html_parts.append(""" </tbody>
292
- </table>
293
- </div>
294
- """)
295
-
296
- html_parts.append("""
297
- </body>
298
- </html>
299
- """)
300
-
301
- return "".join(html_parts)
302
-
303
-
304
- def main():
305
- parser = argparse.ArgumentParser(description="Generate HTML report from run_loop output")
306
- parser.add_argument("input", help="Path to JSON output from run_loop.py (or - for stdin)")
307
- parser.add_argument("-o", "--output", default=None, help="Output HTML file (default: stdout)")
308
- parser.add_argument("--skill-name", default="", help="Skill name to include in the report title")
309
- args = parser.parse_args()
310
-
311
- if args.input == "-":
312
- data = json.load(sys.stdin)
313
- else:
314
- data = json.loads(Path(args.input).read_text())
315
-
316
- html_output = generate_html(data, skill_name=args.skill_name)
317
-
318
- if args.output:
319
- Path(args.output).write_text(html_output)
320
- print(f"Report written to {args.output}", file=sys.stderr)
321
- else:
322
- print(html_output)
323
-
324
-
325
- if __name__ == "__main__":
326
- main()
package/.claude/skills/skill-creator/scripts/improve_description.py DELETED
@@ -1,248 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Improve a skill description based on eval results.
3
-
4
- Takes eval results (from run_eval.py) and generates an improved description
5
- using Claude with extended thinking.
6
- """
7
-
8
- import argparse
9
- import json
10
- import re
11
- import sys
12
- from pathlib import Path
13
-
14
- import anthropic
15
-
16
- from scripts.utils import parse_skill_md
17
-
18
-
19
- def improve_description(
20
- client: anthropic.Anthropic,
21
- skill_name: str,
22
- skill_content: str,
23
- current_description: str,
24
- eval_results: dict,
25
- history: list[dict],
26
- model: str,
27
- test_results: dict | None = None,
28
- log_dir: Path | None = None,
29
- iteration: int | None = None,
30
- ) -> str:
31
- """Call Claude to improve the description based on eval results."""
32
- failed_triggers = [
33
- r for r in eval_results["results"]
34
- if r["should_trigger"] and not r["pass"]
35
- ]
36
- false_triggers = [
37
- r for r in eval_results["results"]
38
- if not r["should_trigger"] and not r["pass"]
39
- ]
40
-
41
- # Build scores summary
42
- train_score = f"{eval_results['summary']['passed']}/{eval_results['summary']['total']}"
43
- if test_results:
44
- test_score = f"{test_results['summary']['passed']}/{test_results['summary']['total']}"
45
- scores_summary = f"Train: {train_score}, Test: {test_score}"
46
- else:
47
- scores_summary = f"Train: {train_score}"
48
-
49
- prompt = f"""You are optimizing a skill description for a Claude Code skill called "{skill_name}". A "skill" is sort of like a prompt, but with progressive disclosure -- there's a title and description that Claude sees when deciding whether to use the skill, and then if it does use the skill, it reads the .md file which has lots more details and potentially links to other resources in the skill folder like helper files and scripts and additional documentation or examples.
50
-
51
- The description appears in Claude's "available_skills" list. When a user sends a query, Claude decides whether to invoke the skill based solely on the title and on this description. Your goal is to write a description that triggers for relevant queries, and doesn't trigger for irrelevant ones.
52
-
53
- Here's the current description:
54
- <current_description>
55
- "{current_description}"
56
- </current_description>
57
-
58
- Current scores ({scores_summary}):
59
- <scores_summary>
60
- """
61
- if failed_triggers:
62
- prompt += "FAILED TO TRIGGER (should have triggered but didn't):\n"
63
- for r in failed_triggers:
64
- prompt += f' - "{r["query"]}" (triggered {r["triggers"]}/{r["runs"]} times)\n'
65
- prompt += "\n"
66
-
67
- if false_triggers:
68
- prompt += "FALSE TRIGGERS (triggered but shouldn't have):\n"
69
- for r in false_triggers:
70
- prompt += f' - "{r["query"]}" (triggered {r["triggers"]}/{r["runs"]} times)\n'
71
- prompt += "\n"
72
-
73
- if history:
74
- prompt += "PREVIOUS ATTEMPTS (do NOT repeat these — try something structurally different):\n\n"
75
- for h in history:
76
- train_s = f"{h.get('train_passed', h.get('passed', 0))}/{h.get('train_total', h.get('total', 0))}"
77
- test_s = f"{h.get('test_passed', '?')}/{h.get('test_total', '?')}" if h.get('test_passed') is not None else None
78
- score_str = f"train={train_s}" + (f", test={test_s}" if test_s else "")
79
- prompt += f'<attempt {score_str}>\n'
80
- prompt += f'Description: "{h["description"]}"\n'
81
- if "results" in h:
82
- prompt += "Train results:\n"
83
- for r in h["results"]:
84
- status = "PASS" if r["pass"] else "FAIL"
85
- prompt += f' [{status}] "{r["query"][:80]}" (triggered {r["triggers"]}/{r["runs"]})\n'
86
- if h.get("note"):
87
- prompt += f'Note: {h["note"]}\n'
88
- prompt += "</attempt>\n\n"
89
-
90
- prompt += f"""</scores_summary>
91
-
92
- Skill content (for context on what the skill does):
93
- <skill_content>
94
- {skill_content}
95
- </skill_content>
96
-
97
- Based on the failures, write a new and improved description that is more likely to trigger correctly. When I say "based on the failures", it's a bit of a tricky line to walk because we don't want to overfit to the specific cases you're seeing. So what I DON'T want you to do is produce an ever-expanding list of specific queries that this skill should or shouldn't trigger for. Instead, try to generalize from the failures to broader categories of user intent and situations where this skill would be useful or not useful. The reason for this is twofold:
98
-
99
- 1. Avoid overfitting
100
- 2. The list might get loooong and it's injected into ALL queries and there might be a lot of skills, so we don't want to blow too much space on any given description.
101
-
102
- Concretely, your description should not be more than about 100-200 words, even if that comes at the cost of accuracy.
103
-
104
- Here are some tips that we've found to work well in writing these descriptions:
105
- - The skill should be phrased in the imperative -- "Use this skill for" rather than "this skill does"
106
- - The skill description should focus on the user's intent, what they are trying to achieve, vs. the implementation details of how the skill works.
107
- - The description competes with other skills for Claude's attention — make it distinctive and immediately recognizable.
108
- - If you're getting lots of failures after repeated attempts, change things up. Try different sentence structures or wordings.
109
-
110
- I'd encourage you to be creative and mix up the style in different iterations since you'll have multiple opportunities to try different approaches and we'll just grab the highest-scoring one at the end.
111
-
112
- Please respond with only the new description text in <new_description> tags, nothing else."""
113
-
114
- response = client.messages.create(
115
- model=model,
116
- max_tokens=16000,
117
- thinking={
118
- "type": "enabled",
119
- "budget_tokens": 10000,
120
- },
121
- messages=[{"role": "user", "content": prompt}],
122
- )
123
-
124
- # Extract thinking and text from response
125
- thinking_text = ""
126
- text = ""
127
- for block in response.content:
128
- if block.type == "thinking":
129
- thinking_text = block.thinking
130
- elif block.type == "text":
131
- text = block.text
132
-
133
- # Parse out the <new_description> tags
134
- match = re.search(r"<new_description>(.*?)</new_description>", text, re.DOTALL)
135
- description = match.group(1).strip().strip('"') if match else text.strip().strip('"')
136
-
137
- # Log the transcript
138
- transcript: dict = {
139
- "iteration": iteration,
140
- "prompt": prompt,
141
- "thinking": thinking_text,
142
- "response": text,
143
- "parsed_description": description,
144
- "char_count": len(description),
145
- "over_limit": len(description) > 1024,
146
- }
147
-
148
- # If over 1024 chars, ask the model to shorten it
149
- if len(description) > 1024:
150
- shorten_prompt = f"Your description is {len(description)} characters, which exceeds the hard 1024 character limit. Please rewrite it to be under 1024 characters while preserving the most important trigger words and intent coverage. Respond with only the new description in <new_description> tags."
151
- shorten_response = client.messages.create(
152
- model=model,
153
- max_tokens=16000,
154
- thinking={
155
- "type": "enabled",
156
- "budget_tokens": 10000,
157
- },
158
- messages=[
159
- {"role": "user", "content": prompt},
160
- {"role": "assistant", "content": text},
161
- {"role": "user", "content": shorten_prompt},
162
- ],
163
- )
164
-
165
- shorten_thinking = ""
166
- shorten_text = ""
167
- for block in shorten_response.content:
168
- if block.type == "thinking":
169
- shorten_thinking = block.thinking
170
- elif block.type == "text":
171
- shorten_text = block.text
172
-
173
- match = re.search(r"<new_description>(.*?)</new_description>", shorten_text, re.DOTALL)
174
- shortened = match.group(1).strip().strip('"') if match else shorten_text.strip().strip('"')
175
-
176
- transcript["rewrite_prompt"] = shorten_prompt
177
- transcript["rewrite_thinking"] = shorten_thinking
178
- transcript["rewrite_response"] = shorten_text
179
- transcript["rewrite_description"] = shortened
180
- transcript["rewrite_char_count"] = len(shortened)
181
- description = shortened
182
-
183
- transcript["final_description"] = description
184
-
185
- if log_dir:
186
- log_dir.mkdir(parents=True, exist_ok=True)
187
- log_file = log_dir / f"improve_iter_{iteration or 'unknown'}.json"
188
- log_file.write_text(json.dumps(transcript, indent=2))
189
-
190
- return description
191
-
192
-
193
- def main():
194
- parser = argparse.ArgumentParser(description="Improve a skill description based on eval results")
195
- parser.add_argument("--eval-results", required=True, help="Path to eval results JSON (from run_eval.py)")
196
- parser.add_argument("--skill-path", required=True, help="Path to skill directory")
197
- parser.add_argument("--history", default=None, help="Path to history JSON (previous attempts)")
198
- parser.add_argument("--model", required=True, help="Model for improvement")
199
- parser.add_argument("--verbose", action="store_true", help="Print thinking to stderr")
200
- args = parser.parse_args()
201
-
202
- skill_path = Path(args.skill_path)
203
- if not (skill_path / "SKILL.md").exists():
204
- print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr)
205
- sys.exit(1)
206
-
207
- eval_results = json.loads(Path(args.eval_results).read_text())
208
- history = []
209
- if args.history:
210
- history = json.loads(Path(args.history).read_text())
211
-
212
- name, _, content = parse_skill_md(skill_path)
213
- current_description = eval_results["description"]
214
-
215
- if args.verbose:
216
- print(f"Current: {current_description}", file=sys.stderr)
217
- print(f"Score: {eval_results['summary']['passed']}/{eval_results['summary']['total']}", file=sys.stderr)
218
-
219
- client = anthropic.Anthropic()
220
- new_description = improve_description(
221
- client=client,
222
- skill_name=name,
223
- skill_content=content,
224
- current_description=current_description,
225
- eval_results=eval_results,
226
- history=history,
227
- model=args.model,
228
- )
229
-
230
- if args.verbose:
231
- print(f"Improved: {new_description}", file=sys.stderr)
232
-
233
- # Output as JSON with both the new description and updated history
234
- output = {
235
- "description": new_description,
236
- "history": history + [{
237
- "description": current_description,
238
- "passed": eval_results["summary"]["passed"],
239
- "failed": eval_results["summary"]["failed"],
240
- "total": eval_results["summary"]["total"],
241
- "results": eval_results["results"],
242
- }],
243
- }
244
- print(json.dumps(output, indent=2))
245
-
246
-
247
- if __name__ == "__main__":
248
- main()