chill-agent 1.5.0 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/contrast-mcp-launcher.mjs +31 -0
- package/.claude-plugin/plugin.json +8 -3
- package/.cline-plugin/plugin.js +6 -0
- package/.codex-plugin/plugin.json +4 -3
- package/.cursor-plugin/marketplace.json +16 -0
- package/.cursor-plugin/plugin.json +2 -2
- package/.devin-plugin/plugin.json +2 -2
- package/.kimi-plugin/plugin.json +11 -29
- package/.omp-plugin/marketplace.json +15 -0
- package/.openclaw/skills/ponytail/SKILL.md +2 -7
- package/.openclaw/skills/ponytail-audit/SKILL.md +3 -6
- package/.openclaw/skills/ponytail-debt/SKILL.md +3 -4
- package/.openclaw/skills/ponytail-gain/SKILL.md +3 -4
- package/.openclaw/skills/ponytail-help/SKILL.md +3 -3
- package/.openclaw/skills/ponytail-review/SKILL.md +3 -6
- package/.opencode/plugins/ponytail.mjs +1 -1
- package/.qoder-plugin/plugin.json +12 -26
- package/AGENTS.md +32 -19
- package/__init__.py +1 -1
- package/assets/antislop-banner.png +0 -0
- package/assets/band/band.png +0 -0
- package/assets/band/board.png +0 -0
- package/assets/benchmark-3model.svg +21 -0
- package/assets/benchmark-agentic.svg +62 -0
- package/assets/codex-icon.png +0 -0
- package/assets/compare/code/after.webp +0 -0
- package/assets/compare/code/before.webp +0 -0
- package/assets/compare/text/after.webp +0 -0
- package/assets/compare/text/before.webp +0 -0
- package/assets/compare/ui/w-all.webp +0 -0
- package/assets/compare/ui/w-antislop.webp +0 -0
- package/assets/compare/ui/w-design.webp +0 -0
- package/assets/compare/ui/without.webp +0 -0
- package/assets/logo-dark.png +0 -0
- package/assets/logo-dark.svg +115 -0
- package/assets/logo-greenpt-dark.svg +27 -0
- package/assets/logo-greenpt.svg +27 -0
- package/assets/logo.png +0 -0
- package/assets/photoreunion-logo-dark.svg +4 -0
- package/assets/photoreunion-logo-light.svg +4 -0
- package/assets/profile.png +0 -0
- package/assets/retriever-logo-dark.svg +4 -0
- package/assets/retriever-logo-light.svg +4 -0
- package/assets/stories-banner.png +0 -0
- package/assets/v5/audit-es.png +0 -0
- package/assets/v5/audit-ja.png +0 -0
- package/assets/v5/audit-ko.png +0 -0
- package/assets/v5/audit-zh-CN.png +0 -0
- package/assets/v5/audit.png +0 -0
- package/assets/v5/beforeafter-es.png +0 -0
- package/assets/v5/beforeafter-ja.png +0 -0
- package/assets/v5/beforeafter-ko.png +0 -0
- package/assets/v5/beforeafter-zh-CN.png +0 -0
- package/assets/v5/beforeafter.png +0 -0
- package/assets/v5/chart-es.png +0 -0
- package/assets/v5/chart-ja.png +0 -0
- package/assets/v5/chart-ko.png +0 -0
- package/assets/v5/chart-zh-CN.png +0 -0
- package/assets/v5/chart.png +0 -0
- package/assets/v5/hero-es.jpg +0 -0
- package/assets/v5/hero-ja.jpg +0 -0
- package/assets/v5/hero-ko.jpg +0 -0
- package/assets/v5/hero-zh-CN.jpg +0 -0
- package/assets/v5/hero.jpg +0 -0
- package/assets/v5/ladder-es.png +0 -0
- package/assets/v5/ladder-ja.png +0 -0
- package/assets/v5/ladder-ko.png +0 -0
- package/assets/v5/ladder-zh-CN.png +0 -0
- package/assets/v5/ladder.png +0 -0
- package/assets/v5/review-es.png +0 -0
- package/assets/v5/review-ja.png +0 -0
- package/assets/v5/review-ko.png +0 -0
- package/assets/v5/review-zh-CN.png +0 -0
- package/assets/v5/review.png +0 -0
- package/assets/v5/tests-es.png +0 -0
- package/assets/v5/tests-ja.png +0 -0
- package/assets/v5/tests-ko.png +0 -0
- package/assets/v5/tests-zh-CN.png +0 -0
- package/assets/v5/tests.png +0 -0
- package/assets/waitlist-banner-es.png +0 -0
- package/assets/waitlist-banner-ja.png +0 -0
- package/assets/waitlist-banner-ko.png +0 -0
- package/assets/waitlist-banner-zh-CN.png +0 -0
- package/assets/waitlist-banner.png +0 -0
- package/benchmarks/.env.example +2 -0
- package/benchmarks/agentic/README.md +201 -0
- package/benchmarks/agentic/answer.py +288 -0
- package/benchmarks/agentic/compare.py +93 -0
- package/benchmarks/agentic/complete.py +178 -0
- package/benchmarks/agentic/judge.py +198 -0
- package/benchmarks/agentic/run.py +543 -0
- package/benchmarks/agentic/tasks.py +1544 -0
- package/benchmarks/arms/baseline.js +2 -0
- package/benchmarks/arms/caveman-SKILL.md +67 -0
- package/benchmarks/arms/caveman.js +8 -0
- package/benchmarks/arms/ponytail.js +8 -0
- package/benchmarks/behavior.js +54 -0
- package/benchmarks/behavior.yaml +40 -0
- package/benchmarks/correctness.js +291 -0
- package/benchmarks/loc.js +17 -19
- package/benchmarks/promptfooconfig.gemini.yaml +32 -0
- package/benchmarks/promptfooconfig.gpt-newest.yaml +33 -0
- package/benchmarks/promptfooconfig.yaml +41 -0
- package/benchmarks/results/2026-06-12-caveman-vs-ponytail.md +71 -0
- package/benchmarks/results/2026-06-12-v4-hardening-vs-caveman.md +125 -0
- package/benchmarks/results/2026-06-15-llama3.2-local.md +82 -0
- package/benchmarks/results/2026-06-16-correctness-gate-fix.md +113 -0
- package/benchmarks/results/2026-06-16-robustness-audit.md +135 -0
- package/benchmarks/results/2026-06-17-agentic-safety.md +164 -0
- package/benchmarks/results/2026-06-17-cost-verification.md +93 -0
- package/benchmarks/results/2026-06-18-agentic.md +219 -0
- package/benchmarks/results/2026-06-22-issue-245-217-comprehension.md +98 -0
- package/benchmarks/results/2026-10-07-agentic.md +186 -0
- package/bin/chill-agent.js +6 -6
- package/bin/chill-agent.mjs +7 -0
- package/cli/index.mjs +241 -0
- package/cli/lib/banner.mjs +21 -0
- package/cli/lib/install.mjs +498 -0
- package/cli/lib/settings.mjs +48 -0
- package/cli/package-lock.json +87 -0
- package/cli/package.json +42 -0
- package/cli/scripts/smoke-test.mjs +95 -0
- package/cli/scripts/smoke-worker.mjs +411 -0
- package/cli/scripts/sync-skills.mjs +81 -0
- package/examples/email-validation.md +156 -0
- package/examples/group-by.md +35 -0
- package/examples/infinite-scroll.md +58 -0
- package/examples/number-formatting.md +37 -0
- package/examples/rate-limit.md +272 -0
- package/examples/react-countdown.md +390 -0
- package/examples/url-params.md +41 -0
- package/gemini-extension.json +2 -2
- package/hooks/band/board.ts +560 -0
- package/hooks/band/logo.ts +3 -0
- package/hooks/band/metrics.ts +311 -0
- package/hooks/band/register.tsx +495 -0
- package/hooks/band/types.d.ts +34 -0
- package/hooks/claude-band.json +6 -0
- package/hooks/claude-codex-hooks.json +34 -4
- package/hooks/copilot-hooks.json +12 -2
- package/hooks/cursor-hooks.json +2 -2
- package/hooks/ponytail-band-import.js +274 -0
- package/hooks/ponytail-config.js +175 -0
- package/hooks/ponytail-map.js +111 -0
- package/hooks/ponytail-subagent.js +94 -0
- package/hooks/qoder-hooks.json +19 -4
- package/i18n/README.ko.md +230 -0
- package/package.json +12 -7
- package/pi-extension/test/extension.test.js +367 -0
- package/pi-extension/test/helpers.test.js +179 -0
- package/plugin.yaml +21 -12
- package/rules/antislop.mdc +18 -0
- package/scripts/build-openclaw-skills.js +60 -0
- package/scripts/check-repo.mjs +389 -0
- package/scripts/publish-openclaw-skills.js +75 -0
- package/scripts/skill-index.mjs +81 -0
- package/skills/antislop/SKILL.md +700 -0
- package/skills/antislop/VERSION +1 -0
- package/skills/antislop-code/SKILL.md +128 -0
- package/skills/antislop-copywriting/SKILL.md +407 -0
- package/skills/antislop-human/LICENSE-web-quality-skills +21 -0
- package/skills/antislop-human/SKILL.md +204 -0
- package/skills/antislop-human/contrast-check.py +168 -0
- package/skills/antislop-human/contrast-mcp.py +121 -0
- package/skills/antislop-human/references/A11Y-PATTERNS.md +249 -0
- package/skills/antislop-human/references/WCAG.md +127 -0
- package/skills/antislop-layoutmobile/SKILL.md +170 -0
- package/skills/antislop-ui/LICENSE-web-quality-skills +21 -0
- package/skills/antislop-ui/SKILL.md +344 -0
- package/skills/antislop-ui/references/CORE-WEB-VITALS.md +181 -0
- package/skills/antislop-ui/references/WEB-BASELINE.md +28 -0
- package/skills/plugin.json +5 -0
- package/src/package.json +3 -0
- package/tests/activation-map.test.js +63 -0
- package/tests/band-import.test.js +111 -0
- package/tests/band.test.tsx +520 -0
- package/tests/behavior.test.js +80 -0
- package/tests/gemini-extension.test.js +2 -2
- package/tests/grok-plugin.test.js +1 -1
- package/tests/hermes-plugin.test.js +1 -1
- package/tests/hooks-windows.test.js +205 -0
- package/tests/kimi-plugin.test.js +1 -1
- package/tests/map.test.js +30 -0
- package/tests/package-scripts.test.js +35 -0
- package/tests/package.test.js +23 -0
- package/tests/uninstall.test.js +202 -0
- /package/tests/{manifests.test.js → manifests.test.mjs} +0 -0
- /package/tests/{rules.test.js → rules.test.mjs} +0 -0
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
# Agentic benchmark: Ponytail 5 vs Ponytail vs no skill
|
|
2
|
+
|
|
3
|
+
*2026-10-07. Opus 5.5, default effort, Claude Code CLI 2.1.288. Headless Claude Code sessions, 39 tasks, 5 runs each.*
|
|
4
|
+
|
|
5
|
+
Ponytail 5 replaces the current Ponytail (v4.13.0). Before it shipped, every change to the rules had to beat the current
|
|
6
|
+
version in a paired run, one change at a time. This file is the final check: Ponytail 5, Ponytail
|
|
7
|
+
(v4.13.0) and the same agent without any skill, measured in one run so that time of day and model
|
|
8
|
+
drift hit all three the same way.
|
|
9
|
+
|
|
10
|
+
## Setup
|
|
11
|
+
|
|
12
|
+
- **Agent:** a headless Claude Code session per task (`claude -p`), Opus 5.5, default effort. The
|
|
13
|
+
skill is loaded with `--plugin-dir`, the same way a user gets it. The no-skill arm loads nothing.
|
|
14
|
+
For every arm I checked the session transcript of a finished cell to prove what was injected.
|
|
15
|
+
This run covers Claude Code; Ponytail also ships for Codex, Cursor, Copilot and more.
|
|
16
|
+
- **Tasks (39):**
|
|
17
|
+
- 12 feature tickets in a real FastAPI + React repo ([full-stack-fastapi-template](https://github.com/fastapi/full-stack-fastapi-template)), scored on the `git diff`.
|
|
18
|
+
- 18 surgical tasks with **hidden checks**: bug fixes, refunds, SQL with untrusted input, path
|
|
19
|
+
traversal, forged tokens, rate limiting, a privacy case (one email to many subscribers without
|
|
20
|
+
exposing their addresses), renames and format changes that must reach every caller.
|
|
21
|
+
- 6 "build me a small app" requests and 3 "show me an example" requests, where the agent picks
|
|
22
|
+
the scope.
|
|
23
|
+
- **Runs:** 5 per task and arm, 585 sessions. The three arms alternate task by task.
|
|
24
|
+
- **Isolation:** every session runs in a fresh temp folder outside any git repo, with `CLAUDE.md`
|
|
25
|
+
loading and auto memory switched off, and only the plugin under test. (An earlier version of the
|
|
26
|
+
harness ran some tasks inside the Ponytail repo. Claude Code then showed the model the repo's
|
|
27
|
+
recent commit messages. That is fixed, and none of the numbers here come from that setup.)
|
|
28
|
+
- **No shell, by design:** Bash is disabled for every arm. Agents write code and tests; the benchmark
|
|
29
|
+
runs them afterwards (hidden checks, injected bugs). This measures what the agent writes, not what
|
|
30
|
+
it can debug.
|
|
31
|
+
|
|
32
|
+
**Statistics.** For each task I take the median of the 5 runs, divide arm by baseline, and report
|
|
33
|
+
the geometric mean of those ratios over the tasks, with a 95% bootstrap interval. "Better on X of Y
|
|
34
|
+
tasks" is a sign test over the tasks.
|
|
35
|
+
|
|
36
|
+
## Results
|
|
37
|
+
|
|
38
|
+
| vs no skill | Ponytail (v4.13) | **Ponytail 5** |
|
|
39
|
+
|---|--:|--:|
|
|
40
|
+
| Lines of code | -48% | **-53%** [-63..-41] |
|
|
41
|
+
| Code information (gzip of the delivered code) | -34% | **-41%** [-51..-30] |
|
|
42
|
+
| Output tokens | -43% | **-45%** [-54..-35] |
|
|
43
|
+
| Cost | -16% [-27..-5] | **-26%** [-36..-16] |
|
|
44
|
+
| Time | -38% | **-41%** [-49..-32] |
|
|
45
|
+
| Turns | -21% | **-22%** |
|
|
46
|
+
| Cheaper on | 25 of 38 tasks | **30 of 38 tasks** |
|
|
47
|
+
|
|
48
|
+
| Ponytail 5 vs Ponytail (v4.13) | change | tasks better / worse |
|
|
49
|
+
|---|--:|--:|
|
|
50
|
+
| Cost | **-10.8%** [-14..-8] | 35 / 4 |
|
|
51
|
+
| Code information (gzip) | **-10.6%** [-15..-6] | 26 / 7 |
|
|
52
|
+
| Lines of code | **-8.6%** [-15..-3] | 20 / 7 |
|
|
53
|
+
| Time | -4.0% [-9..+1] | 23 / 16 |
|
|
54
|
+
| Output tokens | -2.7% [-8..+3] | 21 / 18 |
|
|
55
|
+
| Reply length | +0.1% | 20 / 19 |
|
|
56
|
+
|
|
57
|
+
Mean cost per session: no skill $0.200, Ponytail $0.156, Ponytail 5 $0.142 (list price).
|
|
58
|
+
|
|
59
|
+
### Correctness and safety
|
|
60
|
+
|
|
61
|
+
Only the 18 surgical tasks have hidden checks that run the code (right result and no security or
|
|
62
|
+
privacy hole). On those:
|
|
63
|
+
|
|
64
|
+
| | no skill | Ponytail (v4.13) | Ponytail 5 |
|
|
65
|
+
|---|--:|--:|--:|
|
|
66
|
+
| Hidden checks passed | 86/90 (95.6%) | 86/90 (95.6%) | 87/90 (96.7%) |
|
|
67
|
+
|
|
68
|
+
On the six security tasks of the earlier safety tier (`safe-path`, `sql-user`, `auth-token`,
|
|
69
|
+
`rate-limit`, `csv-sum`, `cache`) all three arms passed 30 of 30 runs.
|
|
70
|
+
|
|
71
|
+
Less code did not cost correctness or safety. It did not add any either: the three arms are level.
|
|
72
|
+
The privacy task is hard for all of them (2 of 5 runs passed in every arm).
|
|
73
|
+
|
|
74
|
+
The other 21 tasks are there to measure code size. Their "pass" only means the code was delivered
|
|
75
|
+
or compiles, so I do not count them as a quality result. (Over all 39 tasks no skill passes 186/195,
|
|
76
|
+
mostly lost to 5 timeouts on one task. That gap says nothing about code quality.)
|
|
77
|
+
|
|
78
|
+
Whether the shorter code on these 21 tasks still does the whole job is checked by the completeness
|
|
79
|
+
judge (`benchmarks/agentic/complete.py`, Opus 5.5 as judge, blind, self-test passed). It reads what
|
|
80
|
+
each run delivered and scores it from 0 (stub) to 3 (fully implements the task).
|
|
81
|
+
|
|
82
|
+
| | no skill | Ponytail 5 |
|
|
83
|
+
|---|--:|--:|
|
|
84
|
+
| Fully implements the task (3 of 3) | 100/100 | 105/105 |
|
|
85
|
+
|
|
86
|
+
The 5 no-skill runs killed at the timeout delivered nothing and are not judged. The judge scored one
|
|
87
|
+
Ponytail 5 run 1 for a missing `col` import; the import is on line 5 of that file (the judge only saw
|
|
88
|
+
the diff), so it is counted as complete.
|
|
89
|
+
|
|
90
|
+
### Tests
|
|
91
|
+
|
|
92
|
+
Ponytail's rule: logic with a branch, a loop, a parser, money or security, or a whole new script,
|
|
93
|
+
leaves one small test or an `assert` self-check behind. Trivial changes need none. I sorted the
|
|
94
|
+
tasks by that rule before measuring (22 need a test, 11 do not, 6 borderline ones count for
|
|
95
|
+
neither), then checked what each agent left behind. To see whether a test is worth anything, I
|
|
96
|
+
injected small bugs (a flipped comparison, a dropped `if`, an off-by-one) into the code the agent
|
|
97
|
+
wrote and ran the agent's own test against each one. The test and mutation tools ship with the
|
|
98
|
+
user-runnable benchmark.
|
|
99
|
+
|
|
100
|
+
| | no skill | Ponytail (v4.13) | Ponytail 5 |
|
|
101
|
+
|---|--:|--:|--:|
|
|
102
|
+
| A test where the logic needs one | 68% | 95% | **98%** |
|
|
103
|
+
| Tests that run (90 cells) | 53 | 84 | **86** |
|
|
104
|
+
| Injected bugs caught, over all cells | 46% | 64% | **66%** |
|
|
105
|
+
| Injected bugs caught, where a test exists | 77% | 69% | 69% |
|
|
106
|
+
| A test where none was needed | 36% | 45% | 45% |
|
|
107
|
+
|
|
108
|
+
No skill writes the most thorough single tests, but often writes none. 20 of the 25 Ponytail 5 cells
|
|
109
|
+
in the last row are two to four `assert` lines inside the learning examples, where they show the
|
|
110
|
+
result; the other 5 add an endpoint test to the repo's existing test suite.
|
|
111
|
+
|
|
112
|
+
### Replies
|
|
113
|
+
|
|
114
|
+
A blind judge (Sonnet 5.5, order shuffled per pair, gated by a self-test) picked the reply that
|
|
115
|
+
serves a busy developer better: true to the diff, tells a reviewer everything they must know, easy
|
|
116
|
+
to read once.
|
|
117
|
+
|
|
118
|
+
| | in the real run (each reply describes its own code) | same code (both replies describe one diff) |
|
|
119
|
+
|---|---|---|
|
|
120
|
+
| Ponytail 5 vs Ponytail (v4.13) | **110 : 67** (18 ties, p=0.002) | **131 : 89** (12 ties, p=0.006) |
|
|
121
|
+
| Ponytail 5 vs no skill | 82 : 106 (7 ties, p=0.09) | 99 : 118 (15 ties, p=0.22) |
|
|
122
|
+
|
|
123
|
+
Ponytail 5 replies are 30% shorter than no skill. Against no skill the judge found no significant
|
|
124
|
+
difference, with a slight lean towards the longer no-skill replies. Against Ponytail (v4.13) it clearly
|
|
125
|
+
prefers Ponytail 5. The main reason in the judge's notes: Ponytail 5 ends with what it skipped or
|
|
126
|
+
did not check and the risks a reviewer should know.
|
|
127
|
+
|
|
128
|
+
## Per task
|
|
129
|
+
|
|
130
|
+
Medians of 5 runs. none = no skill, PT = Ponytail (v4.13), PT5 = Ponytail 5. Lines are the delivered source
|
|
131
|
+
lines (comments included, tests excluded).
|
|
132
|
+
|
|
133
|
+
| task | type | lines: none / PT / PT5 | cost $: none / PT / PT5 | hidden check passed: none / PT / PT5 |
|
|
134
|
+
|---|---|--:|--:|--:|
|
|
135
|
+
| `auth-token` | hidden check | 18 / 17 / 16 | 0.081 / 0.117 / 0.105 | 5/5 / 5/5 / 5/5 |
|
|
136
|
+
| `cache` | hidden check | 11 / 11 / 11 | 0.096 / 0.097 / 0.083 | 5/5 / 5/5 / 5/5 |
|
|
137
|
+
| `conv-seam` | hidden check | 12 / 14 / 14 | 0.122 / 0.149 / 0.139 | 5/5 / 5/5 / 5/5 |
|
|
138
|
+
| `conv-service` | hidden check | 7 / 4 / 4 | 0.099 / 0.110 / 0.089 | 5/5 / 5/5 / 5/5 |
|
|
139
|
+
| `conv-ui` | hidden check | 17 / 17 / 17 | 0.095 / 0.119 / 0.099 | 5/5 / 4/5 / 5/5 |
|
|
140
|
+
| `critic-email` | hidden check | 33 / 14 / 12 | 0.127 / 0.125 / 0.115 | 5/5 / 5/5 / 5/5 |
|
|
141
|
+
| `csv-sum` | hidden check | 58 / 24 / 17 | 0.133 / 0.126 / 0.111 | 5/5 / 5/5 / 5/5 |
|
|
142
|
+
| `rate-limit` | hidden check | 44 / 27 / 26 | 0.147 / 0.145 / 0.144 | 5/5 / 5/5 / 5/5 |
|
|
143
|
+
| `reuse-money` | hidden check | 9 / 9 / 9 | 0.083 / 0.113 / 0.096 | 5/5 / 5/5 / 5/5 |
|
|
144
|
+
| `reuse-slug` | hidden check | 20 / 19 / 21 | 0.091 / 0.118 / 0.103 | 5/5 / 5/5 / 5/5 |
|
|
145
|
+
| `safe-path` | hidden check | 16 / 13 / 13 | 0.117 / 0.113 / 0.106 | 5/5 / 5/5 / 5/5 |
|
|
146
|
+
| `scope-format` | hidden check | 21 / 15 / 15 | 0.137 / 0.153 / 0.147 | 5/5 / 5/5 / 5/5 |
|
|
147
|
+
| `scope-recipients` | hidden check | 13 / 6 / 4 | 0.154 / 0.173 / 0.159 | 2/5 / 2/5 / 2/5 |
|
|
148
|
+
| `scope-rename` | hidden check | 3 / 3 / 3 | 0.133 / 0.163 / 0.128 | 4/5 / 5/5 / 5/5 |
|
|
149
|
+
| `sql-user` | hidden check | 9 / 7 / 7 | 0.093 / 0.099 / 0.089 | 5/5 / 5/5 / 5/5 |
|
|
150
|
+
| `todo-null` | hidden check | 203 / 84 / 77 | 0.166 / 0.163 / 0.125 | 5/5 / 5/5 / 5/5 |
|
|
151
|
+
| `trace-amount` | hidden check | 9 / 9 / 9 | 0.096 / 0.120 / 0.106 | 5/5 / 5/5 / 5/5 |
|
|
152
|
+
| `trace-transfer` | hidden check | 27 / 21 / 20 | 0.135 / 0.133 / 0.126 | 5/5 / 5/5 / 5/5 |
|
|
153
|
+
| `open-dataclass` | learning example | 84 / 62 / 51 | 0.110 / 0.118 / 0.106 | n/a |
|
|
154
|
+
| `open-decorators` | learning example | 79 / 45 / 20 | 0.107 / 0.104 / 0.087 | n/a |
|
|
155
|
+
| `open-mandelbrot` | learning example | 129 / 43 / 32 | 0.237 / 0.114 / 0.113 | n/a |
|
|
156
|
+
| `tmpl-be-archive` | repo feature | 99 / 75 / 59 | 0.353 / 0.319 / 0.269 | n/a |
|
|
157
|
+
| `tmpl-be-bulkdelete` | repo feature | 35 / 22 / 21 | 0.338 / 0.208 / 0.230 | n/a |
|
|
158
|
+
| `tmpl-be-count` | repo feature | 25 / 11 / 12 | 0.280 / 0.172 / 0.145 | n/a |
|
|
159
|
+
| `tmpl-be-csv` | repo feature | 48 / 33 / 38 | 0.270 / 0.226 / 0.218 | n/a |
|
|
160
|
+
| `tmpl-be-duplicate` | repo feature | 20 / 21 / 21 | 0.208 / 0.207 / 0.181 | n/a |
|
|
161
|
+
| `tmpl-be-search` | repo feature | 31 / 29 / 29 | 0.244 / 0.221 / 0.221 | n/a |
|
|
162
|
+
| `tmpl-fe-colorpicker` | repo feature | 135 / 55 / 26 | 0.199 / 0.143 / 0.111 | n/a |
|
|
163
|
+
| `tmpl-fe-command` | repo feature | 319 / 169 / 131 | 0.431 / 0.327 / 0.269 | n/a |
|
|
164
|
+
| `tmpl-fe-datepicker` | repo feature | 335 / 11 / 10 | 0.427 / 0.140 / 0.099 | n/a |
|
|
165
|
+
| `tmpl-fe-dropzone` | repo feature | 247 / 65 / 62 | 0.265 / 0.173 / 0.134 | n/a |
|
|
166
|
+
| `tmpl-fe-rating` | repo feature | 136 / 70 / 60 | 0.183 / 0.152 / 0.124 | n/a |
|
|
167
|
+
| `tmpl-fe-wizard` | repo feature | 311 / 95 / 90 | 0.678 / 0.236 / 0.214 | n/a |
|
|
168
|
+
| `vibe-csvstats` | small app | 124 / 45 / 47 | 0.150 / 0.115 / 0.102 | n/a |
|
|
169
|
+
| `vibe-langgraph` | small app | 148 / 28 / 29 | 0.557 / 0.103 / 0.095 | n/a |
|
|
170
|
+
| `vibe-md2html` | small app | 0 / 106 / 139 | timeout / 0.287 / 0.316 | n/a |
|
|
171
|
+
| `vibe-password` | small app | 224 / 41 / 38 | 0.175 / 0.117 / 0.097 | n/a |
|
|
172
|
+
| `vibe-shortener` | small app | 285 / 83 / 79 | 0.240 / 0.128 / 0.136 | n/a |
|
|
173
|
+
| `vibe-todo` | small app | 197 / 44 / 45 | 0.157 / 0.101 / 0.106 | n/a |
|
|
174
|
+
|
|
175
|
+
## Reproduce
|
|
176
|
+
|
|
177
|
+
See [`benchmarks/agentic/README.md`](../agentic/README.md). Short version: clone the template at
|
|
178
|
+
`cd83fc1`, run `python run.py --selftest` (no API), then
|
|
179
|
+
|
|
180
|
+
```bash
|
|
181
|
+
python run.py --task <the 39 task ids from the table above, comma separated> \
|
|
182
|
+
--arms baseline,ponytail,ponytail2 --models opus --runs 5
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
with `PONYTAIL_PLUGIN_DIR` pointing at Ponytail (v4.13) and `PONYTAIL2_PLUGIN_DIR` at Ponytail 5. Every
|
|
186
|
+
workspace is kept under `runs/<stamp>/`, so any metric can be recomputed offline with `--rescore`.
|
package/bin/chill-agent.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
});
|
|
2
|
+
import('../src/cli.js')
|
|
3
|
+
.then(({ runCli }) => runCli())
|
|
4
|
+
.catch((err) => {
|
|
5
|
+
console.error(err);
|
|
6
|
+
process.exit(1);
|
|
7
|
+
});
|
package/cli/index.mjs
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import path from 'node:path'
|
|
3
|
+
import pc from 'picocolors'
|
|
4
|
+
import { banner } from './lib/banner.mjs'
|
|
5
|
+
import { configureMode } from './lib/settings.mjs'
|
|
6
|
+
import {
|
|
7
|
+
CORE,
|
|
8
|
+
AGENTS,
|
|
9
|
+
atHome,
|
|
10
|
+
skillSourceDir,
|
|
11
|
+
resolveTargets,
|
|
12
|
+
detectAgents,
|
|
13
|
+
detectConflicts,
|
|
14
|
+
detectDuplicateReads,
|
|
15
|
+
detectPluginDoors,
|
|
16
|
+
installSkills,
|
|
17
|
+
installedVersion,
|
|
18
|
+
updateAll,
|
|
19
|
+
updatePointers,
|
|
20
|
+
VERSION,
|
|
21
|
+
} from './lib/install.mjs'
|
|
22
|
+
|
|
23
|
+
// Filled in by loadPrompts() once main() reaches the wizard. --version, --mode and --update
|
|
24
|
+
// never touch them, so they keep working on a Node the prompts cannot load on.
|
|
25
|
+
let intro, outro, select, multiselect, confirm, isCancel, cancel, log, spinner
|
|
26
|
+
|
|
27
|
+
// @clack/core needs util.styleText (Node 20.12.0), so a static import would crash an older
|
|
28
|
+
// Node inside node_modules. Check first, then load; keep the number in step with engines.
|
|
29
|
+
async function loadPrompts() {
|
|
30
|
+
const [major, minor] = process.versions.node.split('.').map(Number)
|
|
31
|
+
if (major < 20 || (major === 20 && minor < 12)) {
|
|
32
|
+
console.error(`antislop: the installer needs Node.js 20.12 or newer, and this is ${process.version}. Update Node, then run it again.`)
|
|
33
|
+
process.exit(1)
|
|
34
|
+
}
|
|
35
|
+
;({ intro, outro, select, multiselect, confirm, isCancel, cancel, log, spinner } = await import('@clack/prompts'))
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const EXTRA_SKILLS = [
|
|
39
|
+
{ value: 'antislop-ui', label: 'antislop-ui', hint: 'UI and visual design rules, with performance mechanisms' },
|
|
40
|
+
{ value: 'antislop-copywriting', label: 'antislop-copywriting', hint: 'copywriting and text rules' },
|
|
41
|
+
{ value: 'antislop-human', label: 'antislop-human', hint: 'accessibility (WCAG 2.2, affordances), with the contrast checker' },
|
|
42
|
+
{ value: 'antislop-layoutmobile', label: 'antislop-layoutmobile', hint: 'responsive layout rules' },
|
|
43
|
+
{ value: 'antislop-code', label: 'antislop-code', hint: 'code comment rules' },
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
function stop(message) {
|
|
47
|
+
cancel(message)
|
|
48
|
+
process.exit(0)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Show the folder the chosen location actually writes. Agents with their own global
|
|
52
|
+
// folder (OpenCode, Antigravity) differ from the project one.
|
|
53
|
+
function displayDir(agent, location) {
|
|
54
|
+
return location === 'global' ? `~/${agent.globalDir ?? agent.dir}` : agent.dir
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// A plugin door keeps its own copy, so --update cannot reach it. Name the command for
|
|
58
|
+
// the ones found rather than send the reader back to a table in the guide.
|
|
59
|
+
function printPluginDoors(found) {
|
|
60
|
+
if (found.length === 0) return
|
|
61
|
+
console.log('\nInstalled as a plugin (these keep their own copy, so updating them is separate):')
|
|
62
|
+
for (const p of found) {
|
|
63
|
+
console.log(` ${p.door.label}${p.version ? ` ${p.version}` : ''}`)
|
|
64
|
+
for (const line of p.door.update.split('\n')) console.log(` ${line}`)
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async function main() {
|
|
69
|
+
// A mistyped mode is a user error, so it gets one line like the terminal guard below
|
|
70
|
+
// rather than the stack trace the top-level catch would print.
|
|
71
|
+
if (process.argv[2] === '--mode') {
|
|
72
|
+
try {
|
|
73
|
+
console.log(configureMode(process.argv.slice(3)))
|
|
74
|
+
} catch (error) {
|
|
75
|
+
console.error(`antislop: ${error.message}`)
|
|
76
|
+
process.exit(1)
|
|
77
|
+
}
|
|
78
|
+
return
|
|
79
|
+
}
|
|
80
|
+
if (process.argv.includes('--version') || process.argv.includes('-v')) {
|
|
81
|
+
console.log(`antislop ${VERSION}`)
|
|
82
|
+
return
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// --update is the non-interactive path, so it runs before the terminal guard and prints
|
|
86
|
+
// plain lines: an update is something a script or a user in a hurry can run unattended.
|
|
87
|
+
if (process.argv.includes('--update')) {
|
|
88
|
+
const { results, pointers } = updateAll()
|
|
89
|
+
if (results.length === 0) {
|
|
90
|
+
console.log(`antislop ${VERSION}: no installed folders found. Nothing to update.`)
|
|
91
|
+
} else {
|
|
92
|
+
for (const r of results) {
|
|
93
|
+
const from = r.from ? `antislop ${r.from}` : 'an older release that records no version'
|
|
94
|
+
console.log(`${r.path}`)
|
|
95
|
+
console.log(` ${from} -> antislop ${VERSION} (${r.skills.length} skill(s) for ${r.agents.join(', ')})`)
|
|
96
|
+
}
|
|
97
|
+
console.log(`\nUpdated ${results.length} folder(s) to antislop ${VERSION}.`)
|
|
98
|
+
if (pointers.length > 0) {
|
|
99
|
+
console.log(`Pointer refreshed in ${pointers.map((p) => path.basename(p)).join(', ')}.`)
|
|
100
|
+
}
|
|
101
|
+
console.log('Start a new agent session for the new rules to load.')
|
|
102
|
+
}
|
|
103
|
+
printPluginDoors(detectPluginDoors())
|
|
104
|
+
return
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// Without a terminal the prompts read EOF at once: the run printed the banner, installed
|
|
108
|
+
// nothing, and still exited 0, so a script around it saw success. Fail loudly instead.
|
|
109
|
+
if (!process.stdin.isTTY) {
|
|
110
|
+
console.error('antislop: this installer needs a terminal. Run `npx antislop-ai` in an interactive shell.')
|
|
111
|
+
process.exit(1)
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
await loadPrompts()
|
|
115
|
+
console.log(banner())
|
|
116
|
+
intro('Install antislop into your agent setup')
|
|
117
|
+
|
|
118
|
+
if (!skillSourceDir()) {
|
|
119
|
+
stop('Could not find the antislop skills. Reinstall the antislop package.')
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
log.step('Choose your skills')
|
|
123
|
+
console.log(` ${pc.green('●')} ${pc.bold(CORE)} ${pc.dim('core rules, always on')}`)
|
|
124
|
+
|
|
125
|
+
const extra = await multiselect({
|
|
126
|
+
message: 'Extra skills to install',
|
|
127
|
+
options: EXTRA_SKILLS,
|
|
128
|
+
required: false,
|
|
129
|
+
})
|
|
130
|
+
if (isCancel(extra)) stop('Install cancelled.')
|
|
131
|
+
const skills = [CORE, ...(extra ?? [])]
|
|
132
|
+
|
|
133
|
+
log.step('Choose where antislop goes')
|
|
134
|
+
const location = await select({
|
|
135
|
+
message: 'Install location',
|
|
136
|
+
options: [
|
|
137
|
+
{ value: 'project', label: 'This project', hint: 'only this folder' },
|
|
138
|
+
{ value: 'global', label: 'Everywhere', hint: 'all your projects' },
|
|
139
|
+
],
|
|
140
|
+
})
|
|
141
|
+
if (isCancel(location)) stop('Install cancelled.')
|
|
142
|
+
|
|
143
|
+
log.step('Choose which agents get antislop')
|
|
144
|
+
const detected = detectAgents(location)
|
|
145
|
+
const chosen = await multiselect({
|
|
146
|
+
message: 'Which agent(s) should antislop install into?',
|
|
147
|
+
options: AGENTS.map((a) => ({
|
|
148
|
+
value: a.id,
|
|
149
|
+
label: a.label,
|
|
150
|
+
hint: detected.includes(a.id) ? `found here (${displayDir(a, location)})` : `will create ${displayDir(a, location)}`,
|
|
151
|
+
})),
|
|
152
|
+
required: 'Pick at least one agent.',
|
|
153
|
+
initialValues: detected,
|
|
154
|
+
})
|
|
155
|
+
if (isCancel(chosen)) stop('Install cancelled.')
|
|
156
|
+
|
|
157
|
+
const targets = resolveTargets(location, chosen)
|
|
158
|
+
await log.message(
|
|
159
|
+
'Installing into:\n' + targets.map((t) => ' ' + t.path).join('\n'),
|
|
160
|
+
{ symbol: pc.cyan('│') }
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
const conflicts = detectConflicts({ skills, targets })
|
|
164
|
+
let overwrite = false
|
|
165
|
+
if (conflicts.length > 0) {
|
|
166
|
+
// Overwriting is how an update happens, and the version on disk is the one fact
|
|
167
|
+
// the user cannot look up anywhere. Name both sides so the choice is obvious.
|
|
168
|
+
const found = [...new Set(targets.map((t) => installedVersion(t.path)).filter(Boolean))]
|
|
169
|
+
const have = found.length > 0 ? `antislop ${found.join(' and ')}` : 'an antislop old enough that it records no version'
|
|
170
|
+
log.warn(`Already here: ${have}. This installer carries ${VERSION}.`)
|
|
171
|
+
|
|
172
|
+
const answer = await select({
|
|
173
|
+
message: `${conflicts.length} skill folder(s) already exist. What should I do?`,
|
|
174
|
+
options: [
|
|
175
|
+
{ value: 'overwrite', label: 'Overwrite them', hint: `update to ${VERSION}` },
|
|
176
|
+
{ value: 'keep', label: 'Keep what is there', hint: 'stay on what is installed' },
|
|
177
|
+
],
|
|
178
|
+
})
|
|
179
|
+
if (isCancel(answer)) stop('Install cancelled.')
|
|
180
|
+
overwrite = answer === 'overwrite'
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const proceed = await confirm({
|
|
184
|
+
message: `Install ${skills.length} skill(s) now?`,
|
|
185
|
+
initialValue: true,
|
|
186
|
+
})
|
|
187
|
+
if (isCancel(proceed) || proceed === false) stop('Install cancelled.')
|
|
188
|
+
|
|
189
|
+
const spin = spinner()
|
|
190
|
+
spin.start('Installing skills...')
|
|
191
|
+
const written = installSkills({ skills, targets, overwrite })
|
|
192
|
+
const pointers = location === 'project' ? updatePointers({ targets, skills }) : []
|
|
193
|
+
spin.stop('Done.')
|
|
194
|
+
|
|
195
|
+
if (pointers.length > 0) {
|
|
196
|
+
log.step('Pointer written to ' + pointers.map((p) => path.basename(p)).join(', '))
|
|
197
|
+
} else if (location === 'project' && atHome()) {
|
|
198
|
+
log.warn('This is your home folder, so no pointer was written: one here would load in every project. Run it inside a project for a pointer.')
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const folderCount = new Set(written.map((w) => path.dirname(w.path))).size
|
|
202
|
+
if (written.length > 0) {
|
|
203
|
+
outro(`Installed ${written.length} skill(s) into ${folderCount} agent folder(s).`)
|
|
204
|
+
console.log(pc.dim('antislop is ready. The next agent session loads it.'))
|
|
205
|
+
} else {
|
|
206
|
+
// "Existing folders were kept" used to read as success. The copies are the
|
|
207
|
+
// old release, so say that plainly and name the choice that fixes it.
|
|
208
|
+
outro('Nothing new to install. Existing folders were kept.')
|
|
209
|
+
console.log(pc.dim('They may be older than this release. To update, run it again and pick "Overwrite them".'))
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// Hermes gates project skills behind a one-time, per-repo trust step, so name it here
|
|
213
|
+
// rather than let the user meet it as a banner after the install looked finished.
|
|
214
|
+
if (location === 'project' && targets.some((t) => t.agents.some((a) => a.id === 'hermes'))) {
|
|
215
|
+
log.warn('Hermes needs one more step: run `hermes skills trust` in this project.')
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// Pi gates its project folders too, but has no command for it: it asks on the first
|
|
219
|
+
// run and `/trust` saves the answer, so a command here would be the wrong instruction.
|
|
220
|
+
if (location === 'project' && targets.some((t) => t.agents.some((a) => a.id === 'pi'))) {
|
|
221
|
+
log.warn('Pi will ask you to trust this project before it loads the skills. `/trust` saves that answer.')
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// Two folders holding the same skill names is how a skill goes missing in OpenCode,
|
|
225
|
+
// which reads both and does not document which copy it keeps.
|
|
226
|
+
for (const dup of detectDuplicateReads({ targets, location })) {
|
|
227
|
+
log.warn(
|
|
228
|
+
`${dup.agent.label} reads both ${dup.paths.join(' and ')}, and antislop is now in both. ` +
|
|
229
|
+
'It may load either copy. Remove one of them to be safe.'
|
|
230
|
+
)
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// Installing through the installer does not update a plugin door the user also has,
|
|
234
|
+
// and nothing else would tell them that copy exists.
|
|
235
|
+
printPluginDoors(detectPluginDoors())
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
main().catch((err) => {
|
|
239
|
+
console.error(err)
|
|
240
|
+
process.exit(1)
|
|
241
|
+
})
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import pc from 'picocolors'
|
|
2
|
+
|
|
3
|
+
const LOGO = [
|
|
4
|
+
' █████╗ ███╗ ██╗████████╗██╗███████╗██╗ ██████╗ ██████╗ ███████╗██╗ ██╗██╗██╗ ██╗ ███████╗',
|
|
5
|
+
'██╔══██╗████╗ ██║╚══██╔══╝██║██╔════╝██║ ██╔═══██╗██╔══██╗ ██╔════╝██║ ██╔╝██║██║ ██║ ██╔════╝',
|
|
6
|
+
'███████║██╔██╗ ██║ ██║ ██║███████╗██║ ██║ ██║██████╔╝ ███████╗█████╔╝ ██║██║ ██║ ███████╗',
|
|
7
|
+
'██╔══██║██║╚██╗██║ ██║ ██║╚════██║██║ ██║ ██║██╔═══╝ ╚════██║██╔═██╗ ██║██║ ██║ ╚════██║',
|
|
8
|
+
'██║ ██║██║ ╚████║ ██║ ██║███████║███████╗╚██████╔╝██║ ███████║██║ ██╗██║███████╗███████╗███████║',
|
|
9
|
+
'╚═╝ ╚═╝╚═╝ ╚═══╝ ╚═╝ ╚═╝╚══════╝╚══════╝ ╚═════╝ ╚═╝ ╚══════╝╚═╝ ╚═╝╚═╝╚══════╝╚══════╝╚══════╝',
|
|
10
|
+
]
|
|
11
|
+
|
|
12
|
+
export function banner() {
|
|
13
|
+
return [
|
|
14
|
+
'',
|
|
15
|
+
...LOGO,
|
|
16
|
+
'',
|
|
17
|
+
' ' + pc.dim('Anti Slop: Rules for AI Coding Agents'),
|
|
18
|
+
' ' + pc.dim('installer v3.2.26'),
|
|
19
|
+
'',
|
|
20
|
+
].join('\n')
|
|
21
|
+
}
|