@dev-sergeev/datalab-kit 0.1.5 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/CHANGELOG.md +11 -0
  2. package/README.md +23 -1
  3. package/THIRD_PARTY_NOTICES.md +3 -0
  4. package/dist/extension.js +2 -0
  5. package/dist/skills.d.ts +2 -0
  6. package/dist/skills.js +54 -0
  7. package/package.json +36 -9
  8. package/skills/init/SKILL.md +31 -0
  9. package/skills/skill-creator/LICENSE.txt +202 -0
  10. package/skills/skill-creator/SKILL.md +610 -0
  11. package/skills/skill-creator/agents/analyzer.md +291 -0
  12. package/skills/skill-creator/agents/comparator.md +213 -0
  13. package/skills/skill-creator/agents/grader.md +234 -0
  14. package/skills/skill-creator/assets/eval_review.html +146 -0
  15. package/skills/skill-creator/eval-viewer/generate_review.py +453 -0
  16. package/skills/skill-creator/eval-viewer/viewer.html +1329 -0
  17. package/skills/skill-creator/references/schemas.md +521 -0
  18. package/skills/skill-creator/scripts/aggregate_benchmark.py +415 -0
  19. package/skills/skill-creator/scripts/generate_report.py +329 -0
  20. package/skills/skill-creator/scripts/improve_description.py +225 -0
  21. package/skills/skill-creator/scripts/package_skill.py +136 -0
  22. package/skills/skill-creator/scripts/pi_runtime.py +177 -0
  23. package/skills/skill-creator/scripts/quick_validate.py +107 -0
  24. package/skills/skill-creator/scripts/run_eval.py +215 -0
  25. package/skills/skill-creator/scripts/run_loop.py +334 -0
  26. package/skills/skill-creator/scripts/utils.py +41 -0
  27. package/node_modules/node-addon-api/common 2.gypi +0 -21
  28. package/node_modules/node-addon-api/except 2.gypi +0 -25
  29. package/node_modules/node-addon-api/napi 2.h +0 -3463
  30. package/node_modules/node-addon-api/napi-inl 2.h +0 -7250
  31. package/node_modules/node-addon-api/napi-inl.deprecated 2.h +0 -186
  32. package/node_modules/node-addon-api/node_addon_api 2.gyp +0 -42
  33. package/node_modules/node-addon-api/node_api 2.gyp +0 -9
  34. package/node_modules/node-addon-api/noexcept 2.gypi +0 -26
  35. package/node_modules/pi-toggle-skills/toggle-skills 2.ts +0 -316
  36. package/node_modules/pi-toggle-skills/vitest.config 2.ts +0 -15
  37. package/node_modules/sprintf-js/.npmignore 2 +0 -1
  38. package/node_modules/sprintf-js/LICENSE 2 +0 -24
  39. package/node_modules/sprintf-js/README 2.md +0 -88
  40. package/node_modules/sprintf-js/bower 2.json +0 -14
  41. package/node_modules/sprintf-js/gruntfile 2.js +0 -36
  42. package/node_modules/sprintf-js/package 2.json +0 -22
  43. package/node_modules/tree-sitter-bash/tree-sitter-bash 2.wasm +0 -0
  44. package/node_modules/typebox/license 2 +0 -23
  45. package/node_modules/typebox/package 2.json +0 -87
  46. package/node_modules/typebox/readme 2.md +0 -370
  47. package/node_modules/zod/compile 2.cjs +0 -96
  48. package/node_modules/zod/compile 2.js +0 -52
  49. package/node_modules/zod/index 2.cjs +0 -52
  50. package/node_modules/zod/index 2.js +0 -7
  51. /package/{node_modules/node-addon-api/nothing 2.c → skills/skill-creator/scripts/__init__.py} +0 -0
@@ -0,0 +1,329 @@
1
+ #!/usr/bin/env python3
2
+ """Generate an HTML report from run_loop.py output.
3
+
4
+ Takes the JSON output from run_loop.py and generates a visual HTML report
5
+ showing each description attempt with check/x for each test case.
6
+ Distinguishes between train and test queries.
7
+
8
+ Changed by Datalab Kit: Pi-specific report copy and reports without held-out queries. Upstream: anthropics/skills
9
+ dbd4588f9e1033efb41dad4bef2f7947c8993d44 (Apache-2.0).
10
+ """
11
+
12
+ import argparse
13
+ import html
14
+ import json
15
+ import sys
16
+ from pathlib import Path
17
+
18
+
19
+ def generate_html(data: dict, auto_refresh: bool = False, skill_name: str = "") -> str:
20
+ """Generate HTML report from loop output data. If auto_refresh is True, adds a meta refresh tag."""
21
+ history = data.get("history", [])
22
+ holdout = data.get("holdout", 0)
23
+ title_prefix = html.escape(skill_name + " \u2014 ") if skill_name else ""
24
+
25
+ # Get all unique queries from train and test sets, with should_trigger info
26
+ train_queries: list[dict] = []
27
+ test_queries: list[dict] = []
28
+ if history:
29
+ for r in history[0].get("train_results", history[0].get("results", [])):
30
+ train_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)})
31
+ if history[0].get("test_results"):
32
+ for r in history[0].get("test_results", []):
33
+ test_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)})
34
+
35
+ refresh_tag = ' <meta http-equiv="refresh" content="5">\n' if auto_refresh else ""
36
+
37
+ html_parts = ["""<!DOCTYPE html>
38
+ <html>
39
+ <head>
40
+ <meta charset="utf-8">
41
+ """ + refresh_tag + """ <title>""" + title_prefix + """Skill Description Optimization</title>
42
+ <link rel="preconnect" href="https://fonts.googleapis.com">
43
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
44
+ <link href="https://fonts.googleapis.com/css2?family=Poppins:wght@500;600&family=Lora:wght@400;500&display=swap" rel="stylesheet">
45
+ <style>
46
+ body {
47
+ font-family: 'Lora', Georgia, serif;
48
+ max-width: 100%;
49
+ margin: 0 auto;
50
+ padding: 20px;
51
+ background: #faf9f5;
52
+ color: #141413;
53
+ }
54
+ h1 { font-family: 'Poppins', sans-serif; color: #141413; }
55
+ .explainer {
56
+ background: white;
57
+ padding: 15px;
58
+ border-radius: 6px;
59
+ margin-bottom: 20px;
60
+ border: 1px solid #e8e6dc;
61
+ color: #b0aea5;
62
+ font-size: 0.875rem;
63
+ line-height: 1.6;
64
+ }
65
+ .summary {
66
+ background: white;
67
+ padding: 15px;
68
+ border-radius: 6px;
69
+ margin-bottom: 20px;
70
+ border: 1px solid #e8e6dc;
71
+ }
72
+ .summary p { margin: 5px 0; }
73
+ .best { color: #788c5d; font-weight: bold; }
74
+ .table-container {
75
+ overflow-x: auto;
76
+ width: 100%;
77
+ }
78
+ table {
79
+ border-collapse: collapse;
80
+ background: white;
81
+ border: 1px solid #e8e6dc;
82
+ border-radius: 6px;
83
+ font-size: 12px;
84
+ min-width: 100%;
85
+ }
86
+ th, td {
87
+ padding: 8px;
88
+ text-align: left;
89
+ border: 1px solid #e8e6dc;
90
+ white-space: normal;
91
+ word-wrap: break-word;
92
+ }
93
+ th {
94
+ font-family: 'Poppins', sans-serif;
95
+ background: #141413;
96
+ color: #faf9f5;
97
+ font-weight: 500;
98
+ }
99
+ th.test-col {
100
+ background: #6a9bcc;
101
+ }
102
+ th.query-col { min-width: 200px; }
103
+ td.description {
104
+ font-family: monospace;
105
+ font-size: 11px;
106
+ word-wrap: break-word;
107
+ max-width: 400px;
108
+ }
109
+ td.result {
110
+ text-align: center;
111
+ font-size: 16px;
112
+ min-width: 40px;
113
+ }
114
+ td.test-result {
115
+ background: #f0f6fc;
116
+ }
117
+ .pass { color: #788c5d; }
118
+ .fail { color: #c44; }
119
+ .rate {
120
+ font-size: 9px;
121
+ color: #b0aea5;
122
+ display: block;
123
+ }
124
+ tr:hover { background: #faf9f5; }
125
+ .score {
126
+ display: inline-block;
127
+ padding: 2px 6px;
128
+ border-radius: 4px;
129
+ font-weight: bold;
130
+ font-size: 11px;
131
+ }
132
+ .score-good { background: #eef2e8; color: #788c5d; }
133
+ .score-ok { background: #fef3c7; color: #d97706; }
134
+ .score-bad { background: #fceaea; color: #c44; }
135
+ .train-label { color: #b0aea5; font-size: 10px; }
136
+ .test-label { color: #6a9bcc; font-size: 10px; font-weight: bold; }
137
+ .best-row { background: #f5f8f2; }
138
+ th.positive-col { border-bottom: 3px solid #788c5d; }
139
+ th.negative-col { border-bottom: 3px solid #c44; }
140
+ th.test-col.positive-col { border-bottom: 3px solid #788c5d; }
141
+ th.test-col.negative-col { border-bottom: 3px solid #c44; }
142
+ .legend { font-family: 'Poppins', sans-serif; display: flex; gap: 20px; margin-bottom: 10px; font-size: 13px; align-items: center; }
143
+ .legend-item { display: flex; align-items: center; gap: 6px; }
144
+ .legend-swatch { width: 16px; height: 16px; border-radius: 3px; display: inline-block; }
145
+ .swatch-positive { background: #141413; border-bottom: 3px solid #788c5d; }
146
+ .swatch-negative { background: #141413; border-bottom: 3px solid #c44; }
147
+ .swatch-test { background: #6a9bcc; }
148
+ .swatch-train { background: #141413; }
149
+ </style>
150
+ </head>
151
+ <body>
152
+ <h1>""" + title_prefix + """Skill Description Optimization</h1>
153
+ <div class="explainer">
154
+ <strong>Optimizing your skill's description.</strong> This page updates automatically as Pi tests different versions of your skill's description. Each row is an iteration — a new description attempt. The columns show test queries: green checkmarks mean the skill triggered correctly (or correctly didn't trigger), red crosses mean it got it wrong. The "Train" score shows performance on queries used to improve the description; the "Test" score shows performance on held-out queries the optimizer hasn't seen. When it's done, return to Pi to apply the best-performing description to your skill.
155
+ </div>
156
+ """]
157
+
158
+ # Summary section
159
+ best_test_score = data.get('best_test_score')
160
+ best_train_score = data.get('best_train_score')
161
+ html_parts.append(f"""
162
+ <div class="summary">
163
+ <p><strong>Original:</strong> {html.escape(data.get('original_description', 'N/A'))}</p>
164
+ <p class="best"><strong>Best:</strong> {html.escape(data.get('best_description', 'N/A'))}</p>
165
+ <p><strong>Best Score:</strong> {data.get('best_score', 'N/A')} {'(test)' if best_test_score else '(train)'}</p>
166
+ <p><strong>Iterations:</strong> {data.get('iterations_run', 0)} | <strong>Train:</strong> {data.get('train_size', '?')} | <strong>Test:</strong> {data.get('test_size', '?')}</p>
167
+ </div>
168
+ """)
169
+
170
+ # Legend
171
+ html_parts.append("""
172
+ <div class="legend">
173
+ <span style="font-weight:600">Query columns:</span>
174
+ <span class="legend-item"><span class="legend-swatch swatch-positive"></span> Should trigger</span>
175
+ <span class="legend-item"><span class="legend-swatch swatch-negative"></span> Should NOT trigger</span>
176
+ <span class="legend-item"><span class="legend-swatch swatch-train"></span> Train</span>
177
+ <span class="legend-item"><span class="legend-swatch swatch-test"></span> Test</span>
178
+ </div>
179
+ """)
180
+
181
+ # Table header
182
+ html_parts.append("""
183
+ <div class="table-container">
184
+ <table>
185
+ <thead>
186
+ <tr>
187
+ <th>Iter</th>
188
+ <th>Train</th>
189
+ <th>Test</th>
190
+ <th class="query-col">Description</th>
191
+ """)
192
+
193
+ # Add column headers for train queries
194
+ for qinfo in train_queries:
195
+ polarity = "positive-col" if qinfo["should_trigger"] else "negative-col"
196
+ html_parts.append(f' <th class="{polarity}">{html.escape(qinfo["query"])}</th>\n')
197
+
198
+ # Add column headers for test queries (different color)
199
+ for qinfo in test_queries:
200
+ polarity = "positive-col" if qinfo["should_trigger"] else "negative-col"
201
+ html_parts.append(f' <th class="test-col {polarity}">{html.escape(qinfo["query"])}</th>\n')
202
+
203
+ html_parts.append(""" </tr>
204
+ </thead>
205
+ <tbody>
206
+ """)
207
+
208
+ # Find best iteration for highlighting
209
+ if test_queries:
210
+ best_iter = max(history, key=lambda h: h.get("test_passed") or 0).get("iteration")
211
+ else:
212
+ best_iter = max(history, key=lambda h: h.get("train_passed", h.get("passed", 0))).get("iteration")
213
+
214
+ # Add rows for each iteration
215
+ for h in history:
216
+ iteration = h.get("iteration", "?")
217
+ train_passed = h.get("train_passed", h.get("passed", 0))
218
+ train_total = h.get("train_total", h.get("total", 0))
219
+ test_passed = h.get("test_passed")
220
+ test_total = h.get("test_total")
221
+ description = h.get("description", "")
222
+ train_results = h.get("train_results", h.get("results", []))
223
+ test_results = h.get("test_results") or []
224
+
225
+ # Create lookups for results by query
226
+ train_by_query = {r["query"]: r for r in train_results}
227
+ test_by_query = {r["query"]: r for r in test_results} if test_results else {}
228
+
229
+ # Compute aggregate correct/total runs across all retries
230
+ def aggregate_runs(results: list[dict]) -> tuple[int, int]:
231
+ correct = 0
232
+ total = 0
233
+ for r in results:
234
+ runs = r.get("runs", 0)
235
+ triggers = r.get("triggers", 0)
236
+ total += runs
237
+ if r.get("should_trigger", True):
238
+ correct += triggers
239
+ else:
240
+ correct += runs - triggers
241
+ return correct, total
242
+
243
+ train_correct, train_runs = aggregate_runs(train_results)
244
+ test_correct, test_runs = aggregate_runs(test_results)
245
+
246
+ # Determine score classes
247
+ def score_class(correct: int, total: int) -> str:
248
+ if total > 0:
249
+ ratio = correct / total
250
+ if ratio >= 0.8:
251
+ return "score-good"
252
+ elif ratio >= 0.5:
253
+ return "score-ok"
254
+ return "score-bad"
255
+
256
+ train_class = score_class(train_correct, train_runs)
257
+ test_class = score_class(test_correct, test_runs)
258
+
259
+ row_class = "best-row" if iteration == best_iter else ""
260
+
261
+ html_parts.append(f""" <tr class="{row_class}">
262
+ <td>{iteration}</td>
263
+ <td><span class="score {train_class}">{train_correct}/{train_runs}</span></td>
264
+ <td><span class="score {test_class}">{test_correct}/{test_runs}</span></td>
265
+ <td class="description">{html.escape(description)}</td>
266
+ """)
267
+
268
+ # Add result for each train query
269
+ for qinfo in train_queries:
270
+ r = train_by_query.get(qinfo["query"], {})
271
+ did_pass = r.get("pass", False)
272
+ triggers = r.get("triggers", 0)
273
+ runs = r.get("runs", 0)
274
+
275
+ icon = "✓" if did_pass else "✗"
276
+ css_class = "pass" if did_pass else "fail"
277
+
278
+ html_parts.append(f' <td class="result {css_class}">{icon}<span class="rate">{triggers}/{runs}</span></td>\n')
279
+
280
+ # Add result for each test query (with different background)
281
+ for qinfo in test_queries:
282
+ r = test_by_query.get(qinfo["query"], {})
283
+ did_pass = r.get("pass", False)
284
+ triggers = r.get("triggers", 0)
285
+ runs = r.get("runs", 0)
286
+
287
+ icon = "✓" if did_pass else "✗"
288
+ css_class = "pass" if did_pass else "fail"
289
+
290
+ html_parts.append(f' <td class="result test-result {css_class}">{icon}<span class="rate">{triggers}/{runs}</span></td>\n')
291
+
292
+ html_parts.append(" </tr>\n")
293
+
294
+ html_parts.append(""" </tbody>
295
+ </table>
296
+ </div>
297
+ """)
298
+
299
+ html_parts.append("""
300
+ </body>
301
+ </html>
302
+ """)
303
+
304
+ return "".join(html_parts)
305
+
306
+
307
+ def main():
308
+ parser = argparse.ArgumentParser(description="Generate HTML report from run_loop output")
309
+ parser.add_argument("input", help="Path to JSON output from run_loop.py (or - for stdin)")
310
+ parser.add_argument("-o", "--output", default=None, help="Output HTML file (default: stdout)")
311
+ parser.add_argument("--skill-name", default="", help="Skill name to include in the report title")
312
+ args = parser.parse_args()
313
+
314
+ if args.input == "-":
315
+ data = json.load(sys.stdin)
316
+ else:
317
+ data = json.loads(Path(args.input).read_text())
318
+
319
+ html_output = generate_html(data, skill_name=args.skill_name)
320
+
321
+ if args.output:
322
+ Path(args.output).write_text(html_output)
323
+ print(f"Report written to {args.output}", file=sys.stderr)
324
+ else:
325
+ print(html_output)
326
+
327
+
328
+ if __name__ == "__main__":
329
+ main()
@@ -0,0 +1,225 @@
1
+ #!/usr/bin/env python3
2
+ """Improve a skill description based on eval results.
3
+
4
+ Takes eval results (from run_eval.py) and generates an improved description
5
+ using the host Pi JSON CLI and its configured provider credentials.
6
+
7
+ Changed by Datalab Kit: isolated Pi backend, model syntax and prompt wording.
8
+ Upstream: anthropics/skills
9
+ dbd4588f9e1033efb41dad4bef2f7947c8993d44 (Apache-2.0).
10
+ """
11
+
12
+ import argparse
13
+ import json
14
+ import re
15
+ import sys
16
+ from pathlib import Path
17
+
18
+ from scripts.utils import parse_skill_md
19
+ from scripts.pi_runtime import call_pi, interrupt_on_termination
20
+
21
+
22
+
23
+
24
+ def improve_description(
25
+ skill_name: str,
26
+ skill_content: str,
27
+ current_description: str,
28
+ eval_results: dict,
29
+ history: list[dict],
30
+ model: str,
31
+ test_results: dict | None = None,
32
+ log_dir: Path | None = None,
33
+ iteration: int | None = None,
34
+ ) -> str:
35
+ """Call Pi to improve the description based on eval results."""
36
+ failed_triggers = [
37
+ r for r in eval_results["results"]
38
+ if r["should_trigger"] and not r["pass"]
39
+ ]
40
+ false_triggers = [
41
+ r for r in eval_results["results"]
42
+ if not r["should_trigger"] and not r["pass"]
43
+ ]
44
+
45
+ # Build scores summary
46
+ train_score = f"{eval_results['summary']['passed']}/{eval_results['summary']['total']}"
47
+ if test_results:
48
+ test_score = f"{test_results['summary']['passed']}/{test_results['summary']['total']}"
49
+ scores_summary = f"Train: {train_score}, Test: {test_score}"
50
+ else:
51
+ scores_summary = f"Train: {train_score}"
52
+
53
+ prompt = f"""You are optimizing a skill description for a Pi skill called "{skill_name}". A "skill" is sort of like a prompt, but with progressive disclosure -- there's a title and description that Pi sees when deciding whether to use the skill, and then if it does use the skill, it reads the .md file which has lots more details and potentially links to other resources in the skill folder like helper files and scripts and additional documentation or examples.
54
+
55
+ The description appears in Pi's "available_skills" list. When a user sends a query, Pi decides whether to read the skill based solely on the title and on this description. Your goal is to write a description that triggers for relevant queries, and doesn't trigger for irrelevant ones.
56
+
57
+ Here's the current description:
58
+ <current_description>
59
+ "{current_description}"
60
+ </current_description>
61
+
62
+ Current scores ({scores_summary}):
63
+ <scores_summary>
64
+ """
65
+ if failed_triggers:
66
+ prompt += "FAILED TO TRIGGER (should have triggered but didn't):\n"
67
+ for r in failed_triggers:
68
+ prompt += f' - "{r["query"]}" (triggered {r["triggers"]}/{r["runs"]} times)\n'
69
+ prompt += "\n"
70
+
71
+ if false_triggers:
72
+ prompt += "FALSE TRIGGERS (triggered but shouldn't have):\n"
73
+ for r in false_triggers:
74
+ prompt += f' - "{r["query"]}" (triggered {r["triggers"]}/{r["runs"]} times)\n'
75
+ prompt += "\n"
76
+
77
+ if history:
78
+ prompt += "PREVIOUS ATTEMPTS (do NOT repeat these — try something structurally different):\n\n"
79
+ for h in history:
80
+ train_s = f"{h.get('train_passed', h.get('passed', 0))}/{h.get('train_total', h.get('total', 0))}"
81
+ test_s = f"{h.get('test_passed', '?')}/{h.get('test_total', '?')}" if h.get('test_passed') is not None else None
82
+ score_str = f"train={train_s}" + (f", test={test_s}" if test_s else "")
83
+ prompt += f'<attempt {score_str}>\n'
84
+ prompt += f'Description: "{h["description"]}"\n'
85
+ if "results" in h:
86
+ prompt += "Train results:\n"
87
+ for r in h["results"]:
88
+ status = "PASS" if r["pass"] else "FAIL"
89
+ prompt += f' [{status}] "{r["query"][:80]}" (triggered {r["triggers"]}/{r["runs"]})\n'
90
+ if h.get("note"):
91
+ prompt += f'Note: {h["note"]}\n'
92
+ prompt += "</attempt>\n\n"
93
+
94
+ prompt += f"""</scores_summary>
95
+
96
+ Skill content (for context on what the skill does):
97
+ <skill_content>
98
+ {skill_content}
99
+ </skill_content>
100
+
101
+ Based on the failures, write a new and improved description that is more likely to trigger correctly. When I say "based on the failures", it's a bit of a tricky line to walk because we don't want to overfit to the specific cases you're seeing. So what I DON'T want you to do is produce an ever-expanding list of specific queries that this skill should or shouldn't trigger for. Instead, try to generalize from the failures to broader categories of user intent and situations where this skill would be useful or not useful. The reason for this is twofold:
102
+
103
+ 1. Avoid overfitting
104
+ 2. The list might get loooong and it's injected into ALL queries and there might be a lot of skills, so we don't want to blow too much space on any given description.
105
+
106
+ Concretely, your description should not be more than about 100-200 words, even if that comes at the cost of accuracy. The Agent Skills format limits descriptions to 1024 characters, so stay comfortably under it.
107
+
108
+ Here are some tips that we've found to work well in writing these descriptions:
109
+ - The skill should be phrased in the imperative -- "Use this skill for" rather than "this skill does"
110
+ - The skill description should focus on the user's intent, what they are trying to achieve, vs. the implementation details of how the skill works.
111
+ - The skill description competes with other skills for Pi's attention — make it distinctive and immediately recognizable.
112
+ - If you're getting lots of failures after repeated attempts, change things up. Try different sentence structures or wordings.
113
+
114
+ I'd encourage you to be creative and mix up the style in different iterations since you'll have multiple opportunities to try different approaches and we'll just grab the highest-scoring one at the end.
115
+
116
+ Please respond with only the new description text in <new_description> tags, nothing else."""
117
+
118
+ text = call_pi(prompt, model)
119
+
120
+ match = re.search(r"<new_description>(.*?)</new_description>", text, re.DOTALL)
121
+ description = match.group(1).strip().strip('"') if match else text.strip().strip('"')
122
+
123
+ transcript: dict = {
124
+ "iteration": iteration,
125
+ "prompt": prompt,
126
+ "response": text,
127
+ "parsed_description": description,
128
+ "char_count": len(description),
129
+ "over_limit": len(description) > 1024,
130
+ }
131
+
132
+ # Safety net: the prompt already states the 1024-char hard limit, but if
133
+ # the model blew past it anyway, make one fresh single-turn call that
134
+ # quotes the too-long version and asks for a shorter rewrite. (The old
135
+ # SDK path did this as a true multi-turn; Pi is called one-shot, so we
136
+ # inline the prior output into the new prompt instead.)
137
+ if len(description) > 1024:
138
+ shorten_prompt = (
139
+ f"{prompt}\n\n"
140
+ f"---\n\n"
141
+ f"A previous attempt produced this description, which at "
142
+ f"{len(description)} characters is over the 1024-character hard limit:\n\n"
143
+ f'"{description}"\n\n'
144
+ f"Rewrite it to be under 1024 characters while keeping the most "
145
+ f"important trigger words and intent coverage. Respond with only "
146
+ f"the new description in <new_description> tags."
147
+ )
148
+ shorten_text = call_pi(shorten_prompt, model)
149
+ match = re.search(r"<new_description>(.*?)</new_description>", shorten_text, re.DOTALL)
150
+ shortened = match.group(1).strip().strip('"') if match else shorten_text.strip().strip('"')
151
+
152
+ transcript["rewrite_prompt"] = shorten_prompt
153
+ transcript["rewrite_response"] = shorten_text
154
+ transcript["rewrite_description"] = shortened
155
+ transcript["rewrite_char_count"] = len(shortened)
156
+ description = shortened
157
+
158
+ transcript["final_description"] = description
159
+
160
+ if log_dir:
161
+ log_dir.mkdir(parents=True, exist_ok=True)
162
+ log_file = log_dir / f"improve_iter_{iteration or 'unknown'}.json"
163
+ log_file.write_text(json.dumps(transcript, indent=2))
164
+
165
+ if not description or len(description) > 1024:
166
+ raise ValueError("Pi must produce a non-empty description of at most 1024 characters")
167
+
168
+ return description
169
+
170
+
171
+ def main():
172
+ parser = argparse.ArgumentParser(description="Improve a skill description based on eval results")
173
+ parser.add_argument("--eval-results", required=True, help="Path to eval results JSON (from run_eval.py)")
174
+ parser.add_argument("--skill-path", required=True, help="Path to skill directory")
175
+ parser.add_argument("--history", default=None, help="Path to history JSON (previous attempts)")
176
+ parser.add_argument("--model", required=True, help="Pi model provider/id[:thinking] for improvement")
177
+ parser.add_argument("--verbose", action="store_true", help="Print thinking to stderr")
178
+ args = parser.parse_args()
179
+
180
+ skill_path = Path(args.skill_path)
181
+ if not (skill_path / "SKILL.md").exists():
182
+ print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr)
183
+ sys.exit(1)
184
+
185
+ eval_results = json.loads(Path(args.eval_results).read_text())
186
+ history = []
187
+ if args.history:
188
+ history = json.loads(Path(args.history).read_text())
189
+
190
+ name, _, content = parse_skill_md(skill_path)
191
+ current_description = eval_results["description"]
192
+
193
+ if args.verbose:
194
+ print(f"Current: {current_description}", file=sys.stderr)
195
+ print(f"Score: {eval_results['summary']['passed']}/{eval_results['summary']['total']}", file=sys.stderr)
196
+
197
+ new_description = improve_description(
198
+ skill_name=name,
199
+ skill_content=content,
200
+ current_description=current_description,
201
+ eval_results=eval_results,
202
+ history=history,
203
+ model=args.model,
204
+ )
205
+
206
+ if args.verbose:
207
+ print(f"Improved: {new_description}", file=sys.stderr)
208
+
209
+ # Output as JSON with both the new description and updated history
210
+ output = {
211
+ "description": new_description,
212
+ "history": history + [{
213
+ "description": current_description,
214
+ "passed": eval_results["summary"]["passed"],
215
+ "failed": eval_results["summary"]["failed"],
216
+ "total": eval_results["summary"]["total"],
217
+ "results": eval_results["results"],
218
+ }],
219
+ }
220
+ print(json.dumps(output, indent=2))
221
+
222
+
223
+ if __name__ == "__main__":
224
+ with interrupt_on_termination():
225
+ main()
@@ -0,0 +1,136 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ Skill Packager - Creates a distributable .skill file of a skill folder
4
+
5
+ Usage:
6
+ python utils/package_skill.py <path/to/skill-folder> [output-directory]
7
+
8
+ Example:
9
+ python utils/package_skill.py skills/public/my-skill
10
+ python utils/package_skill.py skills/public/my-skill ./dist
11
+ """
12
+
13
+ import fnmatch
14
+ import sys
15
+ import zipfile
16
+ from pathlib import Path
17
+ from scripts.quick_validate import validate_skill
18
+
19
+ # Patterns to exclude when packaging skills.
20
+ EXCLUDE_DIRS = {"__pycache__", "node_modules"}
21
+ EXCLUDE_GLOBS = {"*.pyc"}
22
+ EXCLUDE_FILES = {".DS_Store"}
23
+ # Directories excluded only at the skill root (not when nested deeper).
24
+ ROOT_EXCLUDE_DIRS = {"evals"}
25
+
26
+
27
+ def should_exclude(rel_path: Path) -> bool:
28
+ """Check if a path should be excluded from packaging."""
29
+ parts = rel_path.parts
30
+ if any(part in EXCLUDE_DIRS for part in parts):
31
+ return True
32
+ # rel_path is relative to skill_path.parent, so parts[0] is the skill
33
+ # folder name and parts[1] (if present) is the first subdir.
34
+ if len(parts) > 1 and parts[1] in ROOT_EXCLUDE_DIRS:
35
+ return True
36
+ name = rel_path.name
37
+ if name in EXCLUDE_FILES:
38
+ return True
39
+ return any(fnmatch.fnmatch(name, pat) for pat in EXCLUDE_GLOBS)
40
+
41
+
42
+ def package_skill(skill_path, output_dir=None):
43
+ """
44
+ Package a skill folder into a .skill file.
45
+
46
+ Args:
47
+ skill_path: Path to the skill folder
48
+ output_dir: Optional output directory for the .skill file (defaults to current directory)
49
+
50
+ Returns:
51
+ Path to the created .skill file, or None if error
52
+ """
53
+ skill_path = Path(skill_path).resolve()
54
+
55
+ # Validate skill folder exists
56
+ if not skill_path.exists():
57
+ print(f"❌ Error: Skill folder not found: {skill_path}")
58
+ return None
59
+
60
+ if not skill_path.is_dir():
61
+ print(f"❌ Error: Path is not a directory: {skill_path}")
62
+ return None
63
+
64
+ # Validate SKILL.md exists
65
+ skill_md = skill_path / "SKILL.md"
66
+ if not skill_md.exists():
67
+ print(f"❌ Error: SKILL.md not found in {skill_path}")
68
+ return None
69
+
70
+ # Run validation before packaging
71
+ print("🔍 Validating skill...")
72
+ valid, message = validate_skill(skill_path)
73
+ if not valid:
74
+ print(f"❌ Validation failed: {message}")
75
+ print(" Please fix the validation errors before packaging.")
76
+ return None
77
+ print(f"✅ {message}\n")
78
+
79
+ # Determine output location
80
+ skill_name = skill_path.name
81
+ if output_dir:
82
+ output_path = Path(output_dir).resolve()
83
+ output_path.mkdir(parents=True, exist_ok=True)
84
+ else:
85
+ output_path = Path.cwd()
86
+
87
+ skill_filename = output_path / f"{skill_name}.skill"
88
+
89
+ # Create the .skill file (zip format)
90
+ try:
91
+ with zipfile.ZipFile(skill_filename, 'w', zipfile.ZIP_DEFLATED) as zipf:
92
+ # Walk through the skill directory, excluding build artifacts
93
+ for file_path in skill_path.rglob('*'):
94
+ if not file_path.is_file():
95
+ continue
96
+ arcname = file_path.relative_to(skill_path.parent)
97
+ if should_exclude(arcname):
98
+ print(f" Skipped: {arcname}")
99
+ continue
100
+ zipf.write(file_path, arcname)
101
+ print(f" Added: {arcname}")
102
+
103
+ print(f"\n✅ Successfully packaged skill to: {skill_filename}")
104
+ return skill_filename
105
+
106
+ except Exception as e:
107
+ print(f"❌ Error creating .skill file: {e}")
108
+ return None
109
+
110
+
111
+ def main():
112
+ if len(sys.argv) < 2:
113
+ print("Usage: python utils/package_skill.py <path/to/skill-folder> [output-directory]")
114
+ print("\nExample:")
115
+ print(" python utils/package_skill.py skills/public/my-skill")
116
+ print(" python utils/package_skill.py skills/public/my-skill ./dist")
117
+ sys.exit(1)
118
+
119
+ skill_path = sys.argv[1]
120
+ output_dir = sys.argv[2] if len(sys.argv) > 2 else None
121
+
122
+ print(f"📦 Packaging skill: {skill_path}")
123
+ if output_dir:
124
+ print(f" Output directory: {output_dir}")
125
+ print()
126
+
127
+ result = package_skill(skill_path, output_dir)
128
+
129
+ if result:
130
+ sys.exit(0)
131
+ else:
132
+ sys.exit(1)
133
+
134
+
135
+ if __name__ == "__main__":
136
+ main()