pantheon-opencode 1.5.2 → 1.6.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pantheon/code-mode/README.md +4 -2
- package/.pantheon/code-mode/checkpoint_session.py +138 -102
- package/.pantheon/code-mode/compress-inline.py +102 -57
- package/.pantheon/code-mode/manifest.json +2 -8
- package/CHANGELOG.md +59 -0
- package/README.md +115 -55
- package/README.pt-BR.md +86 -43
- package/commands/pantheon-optimize.md +2 -2
- package/package.json +13 -1
- package/plugin.json +1 -1
- package/pyproject.toml +8 -15
- package/scripts/build-agents-md.mjs +2 -2
- package/scripts/check-plugin-v2-coverage.mjs +213 -0
- package/scripts/ci-validate-yaml.py +7 -5
- package/scripts/code_mode_server.py +116 -24
- package/scripts/doctor.mjs +85 -28
- package/scripts/install/cli-ui.mjs +1 -1
- package/scripts/install/config-migration.mjs +124 -3
- package/scripts/install/health-check.mjs +6 -19
- package/scripts/install/interactive.mjs +6 -6
- package/scripts/install/migrate.mjs +1 -1
- package/scripts/install/opencode.mjs +43 -10
- package/scripts/mcp_persistence_server.py +390 -76
- package/scripts/memory_mcp_server.py +370 -196
- package/scripts/postinstall.mjs +0 -7
- package/scripts/probe-context-rehydrate.mjs +28 -11
- package/scripts/prune.mjs +2 -2
- package/scripts/test-opencode-v2-sandbox.sh +1759 -0
- package/scripts/validate-routing.mjs +40 -0
- package/skills-lock.json +9 -10
- package/src/agents/aphrodite.md +0 -1
- package/src/agents/apollo.md +0 -1
- package/src/agents/athena.md +0 -1
- package/src/agents/demeter.md +0 -1
- package/src/agents/gaia.md +0 -1
- package/src/agents/hephaestus.md +0 -1
- package/src/agents/hermes.md +0 -1
- package/src/agents/iris.md +0 -1
- package/src/agents/mnemosyne.md +15 -17
- package/src/agents/nyx.md +0 -1
- package/src/agents/prometheus.md +0 -1
- package/src/agents/talos.md +0 -1
- package/src/agents/themis.md +0 -1
- package/src/agents/zeus.md +1 -3
- package/src/mcp/code_mode_server.py +116 -24
- package/src/mcp/mcp_codemap_module.py +559 -129
- package/src/mcp/mcp_persistence_server.py +390 -76
- package/src/mcp/memory_mcp_server.py +371 -248
- package/src/mcp/pantheon_vision_server.py +136 -24
- package/src/mcp/requirements-mcp.txt +5 -3
- package/src/mcp/requirements-vision.txt +2 -0
- package/src/pantheon/model-command.ts +18 -7
- package/src/pantheon/presets.d.mts +0 -8
- package/src/pantheon/presets.mjs +0 -29
- package/src/pantheon/todo-enforcer.ts +1 -1
- package/src/pantheon/vision.ts +1 -1
- package/src/plugin-v2.ts +134 -104
- package/src/plugins/tui/dist/tui.js +1 -1
- package/src/plugins/tui/package-lock.json +23 -23
- package/src/plugins/tui/package.json +12 -12
- package/src/routing.yml +0 -16
- package/src/skills/README.md +31 -36
- package/src/skills/auto-continue/SKILL.md +27 -77
- package/.pantheon/code-mode/eval-llm-judge.py +0 -255
- package/.pantheon/code-mode/eval-monte-carlo.py +0 -368
- package/.pantheon/code-mode/eval-run.py +0 -197
- package/.pantheon/code-mode/eval-static.py +0 -321
- package/.pantheon/code-mode/session-end-save.py +0 -186
- package/.pantheon/code-mode/session-end-save.sh +0 -18
- package/scripts/recover-zenodo-deposition.mjs +0 -146
- package/scripts/test-opencode-v1-v2-sandbox.sh +0 -862
- package/scripts/validate-zenodo-release.mjs +0 -154
- package/scripts/zenodo-artifact.mjs +0 -94
- package/scripts/zenodo-release-state.mjs +0 -53
- package/src/skills/orchestration-workflow/SKILL.md +0 -311
package/src/skills/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
Skills are reference modules loaded on demand by agents. Each skill provides domain-specific knowledge — architecture patterns, security checklists, optimization strategies, and implementation guides. Agents load skills when their task matches the skill's description, keeping context focused and token-efficient.
|
|
6
6
|
|
|
7
|
-
There are **
|
|
7
|
+
There are **20 skills** divided into **5 domains**.
|
|
8
8
|
|
|
9
9
|
## Skills by Domain
|
|
10
10
|
|
|
@@ -30,75 +30,71 @@ There are **21 skills** divided into **5 domains**.
|
|
|
30
30
|
- Memory bank rules, structure, and optimization — complete guide to Pantheon memory management.
|
|
31
31
|
- Used by: Mnemosyne, Athena
|
|
32
32
|
|
|
33
|
-
6. **
|
|
34
|
-
- Practical step-by-step walkthrough for orchestrating features end-to-end using the multi-agent system, from planning through deployment.
|
|
35
|
-
- Used by: Zeus
|
|
36
|
-
|
|
37
|
-
7. **session-goal** — `session-goal/SKILL.md`
|
|
33
|
+
6. **session-goal** — `session-goal/SKILL.md`
|
|
38
34
|
- Pin session objectives to prevent scope creep across long multi-agent sessions.
|
|
39
35
|
- Used by: Zeus, Mnemosyne
|
|
40
36
|
|
|
41
37
|
### Domain 2: Development Tools
|
|
42
38
|
|
|
43
|
-
|
|
39
|
+
7. **clonedeps** — `clonedeps/SKILL.md`
|
|
44
40
|
- Clones dependency source locally so agents can inspect library internals, understand behavior, and debug integration issues.
|
|
45
41
|
- Used by: Hermes, Hephaestus
|
|
46
42
|
|
|
47
|
-
|
|
43
|
+
8. **git-workflow-and-versioning** — `git-workflow-and-versioning/SKILL.md`
|
|
48
44
|
- Atomic commits, conventional commits, trunk-based development workflow.
|
|
49
45
|
- Used by: Hermes, Prometheus, Iris
|
|
50
46
|
|
|
51
|
-
|
|
47
|
+
9. **incremental-implementation** — `incremental-implementation/SKILL.md`
|
|
52
48
|
- Implement in thin vertical slices — one commit per task, testable, rollback-safe.
|
|
53
49
|
- Used by: Zeus, Hermes, Aphrodite, Demeter, Prometheus, Talos
|
|
54
50
|
|
|
55
|
-
|
|
51
|
+
10. **loop-engineering** — `loop-engineering/SKILL.md`
|
|
56
52
|
- Iterative refinement pattern for complex problems — solve, review, improve, repeat in controlled cycles.
|
|
57
53
|
- Used by: All agents
|
|
58
54
|
|
|
59
|
-
|
|
55
|
+
11. **reflect** — `reflect/SKILL.md`
|
|
60
56
|
- Turns repeated workflow friction into reusable skills, agents, or configuration improvements.
|
|
61
57
|
- Used by: All agents
|
|
62
58
|
|
|
63
|
-
|
|
59
|
+
12. **simplify** — `simplify/SKILL.md`
|
|
64
60
|
- Behavior-preserving code simplification for readability, maintainability, and reduced complexity.
|
|
65
61
|
- Used by: All agents
|
|
66
62
|
|
|
67
|
-
|
|
63
|
+
13. **worktrees** — `worktrees/SKILL.md`
|
|
68
64
|
- Git worktrees as safe, isolated coding lanes for risky or parallel work without affecting the main workspace.
|
|
69
65
|
- Used by: All agents
|
|
70
66
|
|
|
71
67
|
### Domain 3: Quality & Security
|
|
72
68
|
|
|
73
|
-
|
|
69
|
+
14. **code-review-checklist** — `code-review-checklist/SKILL.md`
|
|
74
70
|
- Systematic code review with quality gates, security audit, and parallel checks for structured feedback.
|
|
75
71
|
- Used by: Themis
|
|
76
72
|
|
|
77
|
-
|
|
73
|
+
15. **security-hardening** — `security-hardening/SKILL.md`
|
|
78
74
|
- Security gate — OWASP, SAST, secrets detection, MCP hardening, dependency audit.
|
|
79
75
|
- Used by: Themis, Nyx
|
|
80
76
|
|
|
81
|
-
|
|
77
|
+
16. **tdd-with-agents** — `tdd-with-agents/SKILL.md`
|
|
82
78
|
- TDD enforcement with RED→GREEN→REFACTOR cycle and advanced testing patterns across all layers.
|
|
83
79
|
- Used by: Hermes, Aphrodite, Demeter, Themis, Hephaestus
|
|
84
80
|
|
|
85
81
|
### Domain 4: Planning
|
|
86
82
|
|
|
87
|
-
|
|
83
|
+
17. **codemap** — `codemap/SKILL.md`
|
|
88
84
|
- Generates hierarchical repository maps so agents understand codebase structure without re-reading every file.
|
|
89
85
|
- Used by: Apollo, Zeus
|
|
90
86
|
|
|
91
|
-
|
|
87
|
+
18. **spec-driven-development** — `spec-driven-development/SKILL.md`
|
|
92
88
|
- Define requirements via spec-first before any code — PRD, edge cases, acceptance criteria.
|
|
93
89
|
- Used by: Athena
|
|
94
90
|
|
|
95
|
-
|
|
91
|
+
19. **verification-planning** — `verification-planning/SKILL.md`
|
|
96
92
|
- Plans an evidence path before non-trivial changes, ensuring changes are testable and verifiable.
|
|
97
93
|
- Used by: Athena, Zeus
|
|
98
94
|
|
|
99
95
|
### Domain 5: Frontend Development
|
|
100
96
|
|
|
101
|
-
|
|
97
|
+
20. **visual-review-pipeline** — `visual-review-pipeline/SKILL.md`
|
|
102
98
|
- Automated visual review pipeline — Playwright screenshots, self-analysis, fix loop, escalation.
|
|
103
99
|
- Used by: Aphrodite
|
|
104
100
|
|
|
@@ -130,22 +126,21 @@ The `SKILL.md` file contains YAML frontmatter with `name`, `description`, and pl
|
|
|
130
126
|
| 3 | auto-continue | Orchestration & Workflow | Zeus, Apollo, Hephaestus, Nyx, Gaia |
|
|
131
127
|
| 4 | context-compression | Orchestration & Workflow | Zeus, Mnemosyne, Themis |
|
|
132
128
|
| 5 | memory-bank | Orchestration & Workflow | Mnemosyne, Athena |
|
|
133
|
-
| 6 |
|
|
134
|
-
| 7 |
|
|
135
|
-
| 8 |
|
|
136
|
-
| 9 |
|
|
137
|
-
| 10 |
|
|
138
|
-
| 11 |
|
|
139
|
-
| 12 |
|
|
140
|
-
| 13 |
|
|
141
|
-
| 14 |
|
|
142
|
-
| 15 |
|
|
143
|
-
| 16 |
|
|
144
|
-
| 17 |
|
|
145
|
-
| 18 |
|
|
146
|
-
| 19 |
|
|
147
|
-
| 20 |
|
|
148
|
-
| 21 | visual-review-pipeline | Frontend Development | Aphrodite |
|
|
129
|
+
| 6 | session-goal | Orchestration & Workflow | Zeus, Mnemosyne |
|
|
130
|
+
| 7 | clonedeps | Development Tools | Hermes, Hephaestus |
|
|
131
|
+
| 8 | git-workflow-and-versioning | Development Tools | Hermes, Prometheus, Iris |
|
|
132
|
+
| 9 | incremental-implementation | Development Tools | Zeus, Hermes, Aphrodite, Demeter, Prometheus, Talos |
|
|
133
|
+
| 10 | loop-engineering | Development Tools | All agents |
|
|
134
|
+
| 11 | reflect | Development Tools | All agents |
|
|
135
|
+
| 12 | simplify | Development Tools | All agents |
|
|
136
|
+
| 13 | worktrees | Development Tools | All agents |
|
|
137
|
+
| 14 | code-review-checklist | Quality & Security | Themis |
|
|
138
|
+
| 15 | security-hardening | Quality & Security | Themis, Nyx |
|
|
139
|
+
| 16 | tdd-with-agents | Quality & Security | Hermes, Aphrodite, Demeter, Themis, Hephaestus |
|
|
140
|
+
| 17 | codemap | Planning | Apollo, Zeus |
|
|
141
|
+
| 18 | spec-driven-development | Planning | Athena |
|
|
142
|
+
| 19 | verification-planning | Planning | Athena, Zeus |
|
|
143
|
+
| 20 | visual-review-pipeline | Frontend Development | Aphrodite |
|
|
149
144
|
|
|
150
145
|
---
|
|
151
146
|
|
|
@@ -43,7 +43,7 @@ Disciplined automatic continuation through multi-step tasks. Eliminates unnecess
|
|
|
43
43
|
- A todo requires a decision not covered by the plan
|
|
44
44
|
- An unexpected error changes the approach
|
|
45
45
|
- A dependency is missing or broken
|
|
46
|
-
-
|
|
46
|
+
- Remaining context is too thin to carry the task through — write an explicit handoff instead
|
|
47
47
|
|
|
48
48
|
---
|
|
49
49
|
|
|
@@ -70,7 +70,7 @@ Disciplined automatic continuation through multi-step tasks. Eliminates unnecess
|
|
|
70
70
|
- [ ] Previous step completed successfully (tests pass, no errors)
|
|
71
71
|
- [ ] Next step is within approved plan scope
|
|
72
72
|
- [ ] No new blocking issues emerged
|
|
73
|
-
- [ ]
|
|
73
|
+
- [ ] Remaining context can carry the next step, or an explicit handoff will be written
|
|
74
74
|
|
|
75
75
|
---
|
|
76
76
|
|
|
@@ -104,27 +104,18 @@ Only expand to full cooldown when hitting a mandatory gate.
|
|
|
104
104
|
|
|
105
105
|
## Timeout & Retry Enforcement
|
|
106
106
|
|
|
107
|
-
When a delegated agent does not respond in time, enforce the timeout policy
|
|
108
|
-
|
|
109
|
-
### Behavior by Agent Role
|
|
110
|
-
|
|
111
|
-
| Agent Role | Timeout | Retry Policy | Fallback | Timeout Parcial? |
|
|
112
|
-
|------------|---------|-------------|----------|------------------|
|
|
113
|
-
| Explorer (apollo) | 60s | 2 retries, exponential backoff | athena | ✅ Yes — partial results OK |
|
|
114
|
-
| Implementer (hermes, aphrodite) | 180s | 3 retries, exponential backoff | talos | ❌ No — must produce artifact |
|
|
115
|
-
| Reviewer (themis) | 120s | 2 retries, exponential backoff | zeus | ❌ No — must produce verdict |
|
|
116
|
-
| Infrastructure (prometheus) | 300s | 2 retries, exponential backoff | hermes | ❌ No |
|
|
117
|
-
| Hotfix (talos) | 30s | 1 retry, no backoff | hermes | ✅ Yes — one-liner fix OK |
|
|
107
|
+
When a delegated agent does not respond in time, enforce the timeout policy defined in
|
|
108
|
+
`src/instructions/zeus-timeout-retry.instructions.md`.
|
|
118
109
|
|
|
110
|
+
**That file is the single authority** for per-role timeouts, retry counts, fallback chains, and
|
|
111
|
+
whether partial results are acceptable. It is force-fed into every session via `AGENTS.md`; this
|
|
112
|
+
skill is opt-in. Do not restate those values here — a copy in this skill is precisely what drifted
|
|
113
|
+
out of sync with them. If a dispatch times out, read the role's row from the instruction file.
|
|
119
114
|
|
|
120
115
|
### Timeout Parcial (Partial Results)
|
|
121
116
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
- ✅ @gaia (literature review) — can return partial findings
|
|
125
|
-
- ✅ @talos (hotfix confirmation) — can return without fix if timeout
|
|
126
|
-
- ❌ Never for implementers (@hermes, @aphrodite, @demeter) — must complete or fail
|
|
127
|
-
- ❌ Never for reviewers (@themis) — must produce a verdict
|
|
117
|
+
Whether an agent may return partial results is the **Partial Results OK?** column in that table.
|
|
118
|
+
Only agents marked ✅ there may return partial — never implementers, never reviewers.
|
|
128
119
|
|
|
129
120
|
**How to signal timeout parcial:**
|
|
130
121
|
When dispatching, set the expectation explicitly:
|
|
@@ -167,69 +158,28 @@ Ready for Phase 2? Waiting for go-ahead.
|
|
|
167
158
|
Should I now run the migration tests? [waiting]
|
|
168
159
|
```
|
|
169
160
|
|
|
170
|
-
## Session Heartbeat
|
|
171
|
-
|
|
172
|
-
Auto-save checkpoint every N turns (configurable, default 5).
|
|
173
|
-
|
|
174
|
-
### Heartbeat File
|
|
175
|
-
|
|
176
|
-
`.pantheon/deepwork/<slug>/heartbeat.json`
|
|
161
|
+
## Session Heartbeat & Checkpoints
|
|
177
162
|
|
|
178
|
-
|
|
163
|
+
Session state is **not** file-based. It lives in **pantheon-persistence** under the
|
|
164
|
+
`checkpoint:<slug>` namespace and is cleaned up automatically by a 4h TTL.
|
|
179
165
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
"turn_count": 10,
|
|
185
|
-
"current_phase": 2,
|
|
186
|
-
"status": "alive | warning | stalled | paused | completed",
|
|
187
|
-
"next_action": "brief description of next planned action"
|
|
188
|
-
}
|
|
189
|
-
```
|
|
190
|
-
|
|
191
|
-
### Heartbeat Rules
|
|
192
|
-
1. Update every 5 turns minimum (by default)
|
|
193
|
-
2. Contains NO context — only timestamps and counters
|
|
194
|
-
3. Single file, always overwritten (not versioned)
|
|
195
|
-
4. Used for quick "is this session alive?" checks
|
|
196
|
-
|
|
197
|
-
---
|
|
166
|
+
Write state with `context_save(slug, key, value, session_id=...)` and read it back with
|
|
167
|
+
`context_get(slug, "latest", session_id=...)`. Capture `session_id` from the first `context_save`
|
|
168
|
+
of the session and reuse it for the rest of the session, so the `latest` pointer survives
|
|
169
|
+
compaction.
|
|
198
170
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
Each deepwork task maintains:
|
|
204
|
-
|
|
205
|
-
```
|
|
206
|
-
.pantheon/deepwork/<slug>/
|
|
207
|
-
├── PLAN.md # Immutable plan (created at start)
|
|
208
|
-
├── STATUS.md # Human-readable current state (updated every phase)
|
|
209
|
-
├── heartbeat.json # Lightweight ping (updated every N turns)
|
|
210
|
-
├── checkpoint-<N>.json # Full state snapshot (created at phase boundaries)
|
|
211
|
-
├── session.json # Session metadata (created at start, updated on stop)
|
|
212
|
-
└── REVIEW.md # Final Themis review (created at end)
|
|
213
|
-
```
|
|
214
|
-
|
|
215
|
-
### Checkpoint JSON Schema
|
|
216
|
-
|
|
217
|
-
```json
|
|
218
|
-
{
|
|
219
|
-
"slug": "string — unique task identifier",
|
|
220
|
-
"phase": "integer — current phase number",
|
|
221
|
-
"turn_count": "integer — total turns elapsed",
|
|
222
|
-
"timestamp": "string — ISO 8601 timestamp",
|
|
223
|
-
"context_hash": "string — first 8 hex chars of SHA-256 of STATUS.md",
|
|
224
|
-
"version": "integer — schema version (current: 2)"
|
|
225
|
-
}
|
|
226
|
-
```
|
|
171
|
+
### Heartbeat Check
|
|
172
|
+
- If `context_get(slug, "heartbeat")` returns a checkin older than 300s, log a stall warning and resume
|
|
173
|
+
- Write a heartbeat after every anti-stall recovery action
|
|
227
174
|
|
|
228
175
|
### Checkpoint Rules
|
|
229
|
-
1. Save
|
|
230
|
-
2.
|
|
231
|
-
3.
|
|
232
|
-
|
|
176
|
+
1. Save a checkpoint before **every** delegate dispatch and after every agent returns
|
|
177
|
+
2. Include current phase, turn count, and remaining tasks
|
|
178
|
+
3. On resume: `context_get(slug, "latest")` restores the most recent checkpoint
|
|
179
|
+
|
|
180
|
+
> Do not create `heartbeat.json`, `checkpoint-<N>.json`, or `session.json` under
|
|
181
|
+
> `.pantheon/deepwork/`. That file-based mechanism is retired — TTL handles cleanup, and reading
|
|
182
|
+
> a file that nothing writes back returns stale state.
|
|
233
183
|
|
|
234
184
|
---
|
|
235
185
|
|
|
@@ -1,255 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
# ---
|
|
3
|
-
# description: LLM judge for skill/agent quality — scores 4 dimensions (0-100) via an OpenAI-compatible endpoint
|
|
4
|
-
# timeout: 120
|
|
5
|
-
# ---
|
|
6
|
-
"""LLM judge layer of the plugin-eval certification pipeline.
|
|
7
|
-
|
|
8
|
-
Scores a skill/agent directory on 4 dimensions (correctness, maintainability,
|
|
9
|
-
security, practicality), each 0-100, via ONE structured prompt to an
|
|
10
|
-
OpenAI-compatible chat endpoint. Emits a single JSON document on stdout:
|
|
11
|
-
|
|
12
|
-
{name, dimensions: {correctness, maintainability, security, practicality},
|
|
13
|
-
overall, notes: {<dimension>: <justification>}}
|
|
14
|
-
|
|
15
|
-
Without explicit opt-in it emits a structured skipped result; missing credentials
|
|
16
|
-
or failed calls exit 2 with a clear message.
|
|
17
|
-
|
|
18
|
-
Env vars:
|
|
19
|
-
OPENAI_API_KEY required bearer token
|
|
20
|
-
OPENAI_BASE_URL optional endpoint base (default https://api.openai.com/v1)
|
|
21
|
-
EVAL_JUDGE_MODEL optional model id (default gpt-4o-mini)
|
|
22
|
-
|
|
23
|
-
Stdlib only (urllib.request) — no third-party dependencies.
|
|
24
|
-
"""
|
|
25
|
-
|
|
26
|
-
from __future__ import annotations
|
|
27
|
-
|
|
28
|
-
import argparse
|
|
29
|
-
import json
|
|
30
|
-
import os
|
|
31
|
-
import re
|
|
32
|
-
import sys
|
|
33
|
-
import urllib.error
|
|
34
|
-
import urllib.request
|
|
35
|
-
from pathlib import Path
|
|
36
|
-
from typing import Any
|
|
37
|
-
from urllib.parse import urlparse
|
|
38
|
-
|
|
39
|
-
DEFAULT_MODEL = "gpt-4o-mini"
|
|
40
|
-
DEFAULT_BASE_URL = "https://api.openai.com/v1"
|
|
41
|
-
REQUEST_TIMEOUT = 90.0
|
|
42
|
-
MAX_CONTENT_CHARS = 12_000
|
|
43
|
-
MAX_SUPPORTING_CHARS = 8_000
|
|
44
|
-
MAX_SUPPORTING_FILES = 6
|
|
45
|
-
DIMENSIONS = ("correctness", "maintainability", "security", "practicality")
|
|
46
|
-
|
|
47
|
-
JUDGE_PROMPT = """You are a rigorous quality auditor for AI agent skills and agent definitions used in an agentic coding framework. You decide whether a skill or agent definition is production-quality or AI slop.
|
|
48
|
-
|
|
49
|
-
Apply an explicit ANTI-SLOP bias: flag overengineering, unnecessary abstractions, persona proliferation (multiple fake personas/characters), unverifiable claims, hype language, and instructions that sound impressive but are not actionable.
|
|
50
|
-
|
|
51
|
-
Score the content on 4 dimensions, each 0-100:
|
|
52
|
-
1. correctness — instructions are accurate and internally consistent; no contradictions; no broken references (files, commands, or agents that do not exist or are misnamed).
|
|
53
|
-
2. maintainability — clear structure, single responsibility, no overengineering, YAGNI-respecting, easy to update.
|
|
54
|
-
3. security — no secrets or credentials; no prompt-injection vectors; no dangerous instructions (e.g. rm -rf, curl | bash, disabling safety checks).
|
|
55
|
-
4. practicality — solves a real problem, actionable, not AI slop or hype; a human would actually use it.
|
|
56
|
-
|
|
57
|
-
Return ONLY a JSON object with this exact shape:
|
|
58
|
-
{"correctness": <int 0-100>, "maintainability": <int 0-100>, "security": <int 0-100>, "practicality": <int 0-100>, "notes": {"correctness": "<1-2 sentences>", "maintainability": "<1-2 sentences>", "security": "<1-2 sentences>", "practicality": "<1-2 sentences>"}}
|
|
59
|
-
|
|
60
|
-
Content to evaluate:
|
|
61
|
-
--- BEGIN CONTENT ---
|
|
62
|
-
{content}
|
|
63
|
-
--- END CONTENT ---
|
|
64
|
-
"""
|
|
65
|
-
|
|
66
|
-
_NAME_RE = re.compile(r"^name:\s*[\"']?([^\"'\n]+?)[\"']?\s*$", re.MULTILINE)
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
class JudgeError(Exception):
|
|
70
|
-
"""Expected, user-facing judge failure."""
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
def _frontmatter_name(text: str) -> str | None:
|
|
74
|
-
"""Extract the frontmatter `name` value, or None when absent."""
|
|
75
|
-
if not text.startswith("---"):
|
|
76
|
-
return None
|
|
77
|
-
end = text.find("\n---", 3)
|
|
78
|
-
if end == -1:
|
|
79
|
-
return None
|
|
80
|
-
match = _NAME_RE.search(text[3:end])
|
|
81
|
-
return match.group(1).strip() if match else None
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
def _discover_doc(path: Path) -> Path:
|
|
85
|
-
"""Return the primary doc (SKILL.md or agent .md) for a path."""
|
|
86
|
-
if path.is_file():
|
|
87
|
-
return path
|
|
88
|
-
skill = path / "SKILL.md"
|
|
89
|
-
if skill.is_file():
|
|
90
|
-
return skill
|
|
91
|
-
candidates = sorted(path.glob("*.md"))
|
|
92
|
-
for candidate in candidates:
|
|
93
|
-
try:
|
|
94
|
-
text = candidate.read_text(encoding="utf-8", errors="replace")
|
|
95
|
-
except OSError:
|
|
96
|
-
continue
|
|
97
|
-
if text.startswith("---") and text.find("\n---", 3) != -1:
|
|
98
|
-
return candidate
|
|
99
|
-
if candidates:
|
|
100
|
-
return candidates[0]
|
|
101
|
-
raise JudgeError(f"No SKILL.md or .md file found in {path}")
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
def _supporting_files(doc: Path) -> list[Path]:
|
|
105
|
-
"""Script files next to the doc or in its scripts/ subdir (capped)."""
|
|
106
|
-
roots = [doc.parent]
|
|
107
|
-
scripts_dir = doc.parent / "scripts"
|
|
108
|
-
if scripts_dir.is_dir():
|
|
109
|
-
roots.append(scripts_dir)
|
|
110
|
-
found: list[Path] = []
|
|
111
|
-
for root in roots:
|
|
112
|
-
for pattern in ("*.py", "*.sh", "*.js", "*.ts", "*.mjs"):
|
|
113
|
-
found.extend(sorted(root.glob(pattern)))
|
|
114
|
-
return [p for p in found if p.is_file() and p != doc][:MAX_SUPPORTING_FILES]
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
def _load_content(doc: Path) -> tuple[str, str]:
|
|
118
|
-
"""Return (name, content) with the doc plus small supporting files."""
|
|
119
|
-
try:
|
|
120
|
-
text = doc.read_text(encoding="utf-8", errors="replace")
|
|
121
|
-
except OSError as exc:
|
|
122
|
-
raise JudgeError(f"Cannot read {doc}: {exc}") from exc
|
|
123
|
-
name = _frontmatter_name(text) or doc.stem
|
|
124
|
-
parts = [f"# {name}\n", text[:MAX_CONTENT_CHARS]]
|
|
125
|
-
if len(text) > MAX_CONTENT_CHARS:
|
|
126
|
-
parts.append(f"\n[... truncated {len(text) - MAX_CONTENT_CHARS} chars ...]")
|
|
127
|
-
support: list[str] = []
|
|
128
|
-
total = 0
|
|
129
|
-
for sibling in _supporting_files(doc):
|
|
130
|
-
try:
|
|
131
|
-
data = sibling.read_text(encoding="utf-8", errors="replace")
|
|
132
|
-
except OSError:
|
|
133
|
-
continue
|
|
134
|
-
chunk = f"\n--- supporting file: {sibling.name} ---\n{data[:2000]}"
|
|
135
|
-
if total + len(chunk) > MAX_SUPPORTING_CHARS:
|
|
136
|
-
break
|
|
137
|
-
support.append(chunk)
|
|
138
|
-
total += len(chunk)
|
|
139
|
-
if support:
|
|
140
|
-
parts.append("\n".join(support))
|
|
141
|
-
return name, "".join(parts)
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
def _call_llm(prompt: str, allow_external_llm: bool = False) -> str:
|
|
145
|
-
"""Send one chat completion request via urllib; return the assistant text."""
|
|
146
|
-
if not allow_external_llm:
|
|
147
|
-
raise JudgeError("External LLM calls require --allow-external-llm or PANTHEON_ALLOW_EXTERNAL_LLM=1")
|
|
148
|
-
key = os.getenv("OPENAI_API_KEY")
|
|
149
|
-
if not key:
|
|
150
|
-
raise JudgeError(
|
|
151
|
-
"No LLM available: set OPENAI_API_KEY "
|
|
152
|
-
"(optionally OPENAI_BASE_URL and EVAL_JUDGE_MODEL)."
|
|
153
|
-
)
|
|
154
|
-
base_url = os.getenv("OPENAI_BASE_URL", DEFAULT_BASE_URL).rstrip("/")
|
|
155
|
-
parsed_url = urlparse(base_url)
|
|
156
|
-
if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc:
|
|
157
|
-
raise JudgeError("OPENAI_BASE_URL must use an http or https URL")
|
|
158
|
-
endpoint = base_url + "/chat/completions"
|
|
159
|
-
model = os.getenv("EVAL_JUDGE_MODEL") or DEFAULT_MODEL
|
|
160
|
-
body = {
|
|
161
|
-
"model": model,
|
|
162
|
-
"messages": [{"role": "user", "content": prompt}],
|
|
163
|
-
"temperature": 0.2,
|
|
164
|
-
}
|
|
165
|
-
request = urllib.request.Request(
|
|
166
|
-
endpoint,
|
|
167
|
-
data=json.dumps(body).encode("utf-8"),
|
|
168
|
-
headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
|
|
169
|
-
method="POST",
|
|
170
|
-
)
|
|
171
|
-
try:
|
|
172
|
-
with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT) as response:
|
|
173
|
-
payload = json.loads(response.read().decode("utf-8"))
|
|
174
|
-
except urllib.error.HTTPError as exc:
|
|
175
|
-
detail = exc.read().decode("utf-8", errors="replace")[:200].replace(key, "[redacted]")
|
|
176
|
-
raise JudgeError(f"LLM endpoint returned HTTP {exc.code}: {detail}") from exc
|
|
177
|
-
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
|
178
|
-
reason = getattr(exc, "reason", exc)
|
|
179
|
-
raise JudgeError(f"LLM request failed: {reason}") from exc
|
|
180
|
-
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
|
181
|
-
raise JudgeError(f"LLM returned invalid JSON payload: {exc}") from exc
|
|
182
|
-
choices = payload.get("choices") if isinstance(payload, dict) else None
|
|
183
|
-
message = choices[0].get("message") if choices else None
|
|
184
|
-
text = message.get("content") if isinstance(message, dict) else None
|
|
185
|
-
if not isinstance(text, str) or not text.strip():
|
|
186
|
-
raise JudgeError("LLM returned an empty response.")
|
|
187
|
-
return text.replace(key, "[redacted]")
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
def _parse_json_object(value: str) -> dict[str, Any] | None:
|
|
191
|
-
"""Parse strict JSON and the fenced JSON commonly returned by models."""
|
|
192
|
-
candidates = [value.strip()]
|
|
193
|
-
if candidates[0].startswith("```") and candidates[0].endswith("```"):
|
|
194
|
-
candidates.append(candidates[0].split("\n", 1)[-1][:-3].strip())
|
|
195
|
-
for candidate in candidates:
|
|
196
|
-
try:
|
|
197
|
-
parsed = json.loads(candidate)
|
|
198
|
-
except json.JSONDecodeError:
|
|
199
|
-
continue
|
|
200
|
-
if isinstance(parsed, dict):
|
|
201
|
-
return parsed
|
|
202
|
-
return None
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
def _parse_scores(text: str) -> dict[str, Any]:
|
|
206
|
-
"""Validate the LLM JSON into the report's dimensions/overall/notes."""
|
|
207
|
-
parsed = _parse_json_object(text)
|
|
208
|
-
if parsed is None:
|
|
209
|
-
raise JudgeError("LLM did not return valid JSON.")
|
|
210
|
-
dimensions: dict[str, int] = {}
|
|
211
|
-
for dim in DIMENSIONS:
|
|
212
|
-
raw = parsed.get(dim)
|
|
213
|
-
try:
|
|
214
|
-
value = int(raw)
|
|
215
|
-
except (TypeError, ValueError) as exc:
|
|
216
|
-
raise JudgeError(f"LLM returned non-integer score for {dim}: {raw!r}") from exc
|
|
217
|
-
if not 0 <= value <= 100:
|
|
218
|
-
raise JudgeError(f"LLM returned out-of-range score for {dim}: {value}")
|
|
219
|
-
dimensions[dim] = value
|
|
220
|
-
raw_notes = parsed.get("notes")
|
|
221
|
-
notes = (
|
|
222
|
-
{dim: str(raw_notes.get(dim, "")) for dim in DIMENSIONS}
|
|
223
|
-
if isinstance(raw_notes, dict)
|
|
224
|
-
else {dim: "" for dim in DIMENSIONS}
|
|
225
|
-
)
|
|
226
|
-
overall = round(sum(dimensions.values()) / len(dimensions), 1)
|
|
227
|
-
return {"dimensions": dimensions, "overall": overall, "notes": notes}
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
def main(argv: list[str] | None = None) -> int:
|
|
231
|
-
"""CLI entry point: `python eval-llm-judge.py <skill-or-agent-dir>`."""
|
|
232
|
-
parser = argparse.ArgumentParser(
|
|
233
|
-
description="LLM judge for skill/agent quality (plugin-eval layer 2)."
|
|
234
|
-
)
|
|
235
|
-
parser.add_argument("path", help="Path to a skill/agent directory or doc file")
|
|
236
|
-
parser.add_argument("--allow-external-llm", action="store_true", help="Permit sending content to the configured LLM endpoint")
|
|
237
|
-
args = parser.parse_args(argv)
|
|
238
|
-
allow_external_llm = args.allow_external_llm or os.getenv("PANTHEON_ALLOW_EXTERNAL_LLM") == "1"
|
|
239
|
-
try:
|
|
240
|
-
doc = _discover_doc(Path(args.path))
|
|
241
|
-
name, content = _load_content(doc)
|
|
242
|
-
if not allow_external_llm:
|
|
243
|
-
print(json.dumps({"name": name, "skipped": True, "reason": "external LLM calls not opted in"}, ensure_ascii=False, indent=2))
|
|
244
|
-
return 0
|
|
245
|
-
raw = _call_llm(JUDGE_PROMPT.replace("{content}", content), allow_external_llm=True)
|
|
246
|
-
result = _parse_scores(raw)
|
|
247
|
-
except JudgeError as exc:
|
|
248
|
-
print(f"eval-llm-judge: {exc}", file=sys.stderr)
|
|
249
|
-
return 2
|
|
250
|
-
print(json.dumps({"name": name, **result}, ensure_ascii=False, indent=2))
|
|
251
|
-
return 0
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
if __name__ == "__main__":
|
|
255
|
-
sys.exit(main())
|