pantheon-opencode 1.5.2 → 1.6.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/.pantheon/code-mode/README.md +4 -2
  2. package/.pantheon/code-mode/checkpoint_session.py +138 -102
  3. package/.pantheon/code-mode/compress-inline.py +102 -57
  4. package/.pantheon/code-mode/manifest.json +2 -8
  5. package/CHANGELOG.md +59 -0
  6. package/README.md +115 -55
  7. package/README.pt-BR.md +86 -43
  8. package/commands/pantheon-optimize.md +2 -2
  9. package/package.json +13 -1
  10. package/plugin.json +1 -1
  11. package/pyproject.toml +8 -15
  12. package/scripts/build-agents-md.mjs +2 -2
  13. package/scripts/check-plugin-v2-coverage.mjs +213 -0
  14. package/scripts/ci-validate-yaml.py +7 -5
  15. package/scripts/code_mode_server.py +116 -24
  16. package/scripts/doctor.mjs +85 -28
  17. package/scripts/install/cli-ui.mjs +1 -1
  18. package/scripts/install/config-migration.mjs +124 -3
  19. package/scripts/install/health-check.mjs +6 -19
  20. package/scripts/install/interactive.mjs +6 -6
  21. package/scripts/install/migrate.mjs +1 -1
  22. package/scripts/install/opencode.mjs +43 -10
  23. package/scripts/mcp_persistence_server.py +390 -76
  24. package/scripts/memory_mcp_server.py +370 -196
  25. package/scripts/postinstall.mjs +0 -7
  26. package/scripts/probe-context-rehydrate.mjs +28 -11
  27. package/scripts/prune.mjs +2 -2
  28. package/scripts/test-opencode-v2-sandbox.sh +1759 -0
  29. package/scripts/validate-routing.mjs +40 -0
  30. package/skills-lock.json +9 -10
  31. package/src/agents/aphrodite.md +0 -1
  32. package/src/agents/apollo.md +0 -1
  33. package/src/agents/athena.md +0 -1
  34. package/src/agents/demeter.md +0 -1
  35. package/src/agents/gaia.md +0 -1
  36. package/src/agents/hephaestus.md +0 -1
  37. package/src/agents/hermes.md +0 -1
  38. package/src/agents/iris.md +0 -1
  39. package/src/agents/mnemosyne.md +15 -17
  40. package/src/agents/nyx.md +0 -1
  41. package/src/agents/prometheus.md +0 -1
  42. package/src/agents/talos.md +0 -1
  43. package/src/agents/themis.md +0 -1
  44. package/src/agents/zeus.md +1 -3
  45. package/src/mcp/code_mode_server.py +116 -24
  46. package/src/mcp/mcp_codemap_module.py +559 -129
  47. package/src/mcp/mcp_persistence_server.py +390 -76
  48. package/src/mcp/memory_mcp_server.py +371 -248
  49. package/src/mcp/pantheon_vision_server.py +136 -24
  50. package/src/mcp/requirements-mcp.txt +5 -3
  51. package/src/mcp/requirements-vision.txt +2 -0
  52. package/src/pantheon/model-command.ts +18 -7
  53. package/src/pantheon/presets.d.mts +0 -8
  54. package/src/pantheon/presets.mjs +0 -29
  55. package/src/pantheon/todo-enforcer.ts +1 -1
  56. package/src/pantheon/vision.ts +1 -1
  57. package/src/plugin-v2.ts +134 -104
  58. package/src/plugins/tui/dist/tui.js +1 -1
  59. package/src/plugins/tui/package-lock.json +23 -23
  60. package/src/plugins/tui/package.json +12 -12
  61. package/src/routing.yml +0 -16
  62. package/src/skills/README.md +31 -36
  63. package/src/skills/auto-continue/SKILL.md +27 -77
  64. package/.pantheon/code-mode/eval-llm-judge.py +0 -255
  65. package/.pantheon/code-mode/eval-monte-carlo.py +0 -368
  66. package/.pantheon/code-mode/eval-run.py +0 -197
  67. package/.pantheon/code-mode/eval-static.py +0 -321
  68. package/.pantheon/code-mode/session-end-save.py +0 -186
  69. package/.pantheon/code-mode/session-end-save.sh +0 -18
  70. package/scripts/recover-zenodo-deposition.mjs +0 -146
  71. package/scripts/test-opencode-v1-v2-sandbox.sh +0 -862
  72. package/scripts/validate-zenodo-release.mjs +0 -154
  73. package/scripts/zenodo-artifact.mjs +0 -94
  74. package/scripts/zenodo-release-state.mjs +0 -53
  75. package/src/skills/orchestration-workflow/SKILL.md +0 -311
@@ -4,7 +4,7 @@
4
4
 
5
5
  Skills are reference modules loaded on demand by agents. Each skill provides domain-specific knowledge — architecture patterns, security checklists, optimization strategies, and implementation guides. Agents load skills when their task matches the skill's description, keeping context focused and token-efficient.
6
6
 
7
- There are **21 skills** divided into **5 domains**.
7
+ There are **20 skills** divided into **5 domains**.
8
8
 
9
9
  ## Skills by Domain
10
10
 
@@ -30,75 +30,71 @@ There are **21 skills** divided into **5 domains**.
30
30
  - Memory bank rules, structure, and optimization — complete guide to Pantheon memory management.
31
31
  - Used by: Mnemosyne, Athena
32
32
 
33
- 6. **orchestration-workflow** — `orchestration-workflow/SKILL.md`
34
- - Practical step-by-step walkthrough for orchestrating features end-to-end using the multi-agent system, from planning through deployment.
35
- - Used by: Zeus
36
-
37
- 7. **session-goal** — `session-goal/SKILL.md`
33
+ 6. **session-goal** — `session-goal/SKILL.md`
38
34
  - Pin session objectives to prevent scope creep across long multi-agent sessions.
39
35
  - Used by: Zeus, Mnemosyne
40
36
 
41
37
  ### Domain 2: Development Tools
42
38
 
43
- 8. **clonedeps** — `clonedeps/SKILL.md`
39
+ 7. **clonedeps** — `clonedeps/SKILL.md`
44
40
  - Clones dependency source locally so agents can inspect library internals, understand behavior, and debug integration issues.
45
41
  - Used by: Hermes, Hephaestus
46
42
 
47
- 9. **git-workflow-and-versioning** — `git-workflow-and-versioning/SKILL.md`
43
+ 8. **git-workflow-and-versioning** — `git-workflow-and-versioning/SKILL.md`
48
44
  - Atomic commits, conventional commits, trunk-based development workflow.
49
45
  - Used by: Hermes, Prometheus, Iris
50
46
 
51
- 10. **incremental-implementation** — `incremental-implementation/SKILL.md`
47
+ 9. **incremental-implementation** — `incremental-implementation/SKILL.md`
52
48
  - Implement in thin vertical slices — one commit per task, testable, rollback-safe.
53
49
  - Used by: Zeus, Hermes, Aphrodite, Demeter, Prometheus, Talos
54
50
 
55
- 11. **loop-engineering** — `loop-engineering/SKILL.md`
51
+ 10. **loop-engineering** — `loop-engineering/SKILL.md`
56
52
  - Iterative refinement pattern for complex problems — solve, review, improve, repeat in controlled cycles.
57
53
  - Used by: All agents
58
54
 
59
- 12. **reflect** — `reflect/SKILL.md`
55
+ 11. **reflect** — `reflect/SKILL.md`
60
56
  - Turns repeated workflow friction into reusable skills, agents, or configuration improvements.
61
57
  - Used by: All agents
62
58
 
63
- 13. **simplify** — `simplify/SKILL.md`
59
+ 12. **simplify** — `simplify/SKILL.md`
64
60
  - Behavior-preserving code simplification for readability, maintainability, and reduced complexity.
65
61
  - Used by: All agents
66
62
 
67
- 14. **worktrees** — `worktrees/SKILL.md`
63
+ 13. **worktrees** — `worktrees/SKILL.md`
68
64
  - Git worktrees as safe, isolated coding lanes for risky or parallel work without affecting the main workspace.
69
65
  - Used by: All agents
70
66
 
71
67
  ### Domain 3: Quality & Security
72
68
 
73
- 15. **code-review-checklist** — `code-review-checklist/SKILL.md`
69
+ 14. **code-review-checklist** — `code-review-checklist/SKILL.md`
74
70
  - Systematic code review with quality gates, security audit, and parallel checks for structured feedback.
75
71
  - Used by: Themis
76
72
 
77
- 16. **security-hardening** — `security-hardening/SKILL.md`
73
+ 15. **security-hardening** — `security-hardening/SKILL.md`
78
74
  - Security gate — OWASP, SAST, secrets detection, MCP hardening, dependency audit.
79
75
  - Used by: Themis, Nyx
80
76
 
81
- 17. **tdd-with-agents** — `tdd-with-agents/SKILL.md`
77
+ 16. **tdd-with-agents** — `tdd-with-agents/SKILL.md`
82
78
  - TDD enforcement with RED→GREEN→REFACTOR cycle and advanced testing patterns across all layers.
83
79
  - Used by: Hermes, Aphrodite, Demeter, Themis, Hephaestus
84
80
 
85
81
  ### Domain 4: Planning
86
82
 
87
- 18. **codemap** — `codemap/SKILL.md`
83
+ 17. **codemap** — `codemap/SKILL.md`
88
84
  - Generates hierarchical repository maps so agents understand codebase structure without re-reading every file.
89
85
  - Used by: Apollo, Zeus
90
86
 
91
- 19. **spec-driven-development** — `spec-driven-development/SKILL.md`
87
+ 18. **spec-driven-development** — `spec-driven-development/SKILL.md`
92
88
  - Define requirements via spec-first before any code — PRD, edge cases, acceptance criteria.
93
89
  - Used by: Athena
94
90
 
95
- 20. **verification-planning** — `verification-planning/SKILL.md`
91
+ 19. **verification-planning** — `verification-planning/SKILL.md`
96
92
  - Plans an evidence path before non-trivial changes, ensuring changes are testable and verifiable.
97
93
  - Used by: Athena, Zeus
98
94
 
99
95
  ### Domain 5: Frontend Development
100
96
 
101
- 21. **visual-review-pipeline** — `visual-review-pipeline/SKILL.md`
97
+ 20. **visual-review-pipeline** — `visual-review-pipeline/SKILL.md`
102
98
  - Automated visual review pipeline — Playwright screenshots, self-analysis, fix loop, escalation.
103
99
  - Used by: Aphrodite
104
100
 
@@ -130,22 +126,21 @@ The `SKILL.md` file contains YAML frontmatter with `name`, `description`, and pl
130
126
  | 3 | auto-continue | Orchestration & Workflow | Zeus, Apollo, Hephaestus, Nyx, Gaia |
131
127
  | 4 | context-compression | Orchestration & Workflow | Zeus, Mnemosyne, Themis |
132
128
  | 5 | memory-bank | Orchestration & Workflow | Mnemosyne, Athena |
133
- | 6 | orchestration-workflow | Orchestration & Workflow | Zeus |
134
- | 7 | session-goal | Orchestration & Workflow | Zeus, Mnemosyne |
135
- | 8 | clonedeps | Development Tools | Hermes, Hephaestus |
136
- | 9 | git-workflow-and-versioning | Development Tools | Hermes, Prometheus, Iris |
137
- | 10 | incremental-implementation | Development Tools | Zeus, Hermes, Aphrodite, Demeter, Prometheus, Talos |
138
- | 11 | loop-engineering | Development Tools | All agents |
139
- | 12 | reflect | Development Tools | All agents |
140
- | 13 | simplify | Development Tools | All agents |
141
- | 14 | worktrees | Development Tools | All agents |
142
- | 15 | code-review-checklist | Quality & Security | Themis |
143
- | 16 | security-hardening | Quality & Security | Themis, Nyx |
144
- | 17 | tdd-with-agents | Quality & Security | Hermes, Aphrodite, Demeter, Themis, Hephaestus |
145
- | 18 | codemap | Planning | Apollo, Zeus |
146
- | 19 | spec-driven-development | Planning | Athena |
147
- | 20 | verification-planning | Planning | Athena, Zeus |
148
- | 21 | visual-review-pipeline | Frontend Development | Aphrodite |
129
+ | 6 | session-goal | Orchestration & Workflow | Zeus, Mnemosyne |
130
+ | 7 | clonedeps | Development Tools | Hermes, Hephaestus |
131
+ | 8 | git-workflow-and-versioning | Development Tools | Hermes, Prometheus, Iris |
132
+ | 9 | incremental-implementation | Development Tools | Zeus, Hermes, Aphrodite, Demeter, Prometheus, Talos |
133
+ | 10 | loop-engineering | Development Tools | All agents |
134
+ | 11 | reflect | Development Tools | All agents |
135
+ | 12 | simplify | Development Tools | All agents |
136
+ | 13 | worktrees | Development Tools | All agents |
137
+ | 14 | code-review-checklist | Quality & Security | Themis |
138
+ | 15 | security-hardening | Quality & Security | Themis, Nyx |
139
+ | 16 | tdd-with-agents | Quality & Security | Hermes, Aphrodite, Demeter, Themis, Hephaestus |
140
+ | 17 | codemap | Planning | Apollo, Zeus |
141
+ | 18 | spec-driven-development | Planning | Athena |
142
+ | 19 | verification-planning | Planning | Athena, Zeus |
143
+ | 20 | visual-review-pipeline | Frontend Development | Aphrodite |
149
144
 
150
145
  ---
151
146
 
@@ -43,7 +43,7 @@ Disciplined automatic continuation through multi-step tasks. Eliminates unnecess
43
43
  - A todo requires a decision not covered by the plan
44
44
  - An unexpected error changes the approach
45
45
  - A dependency is missing or broken
46
- - A task would exceed remaining `steps` budget
46
+ - Remaining context is too thin to carry the task through — write an explicit handoff instead
47
47
 
48
48
  ---
49
49
 
@@ -70,7 +70,7 @@ Disciplined automatic continuation through multi-step tasks. Eliminates unnecess
70
70
  - [ ] Previous step completed successfully (tests pass, no errors)
71
71
  - [ ] Next step is within approved plan scope
72
72
  - [ ] No new blocking issues emerged
73
- - [ ] `steps` counter has sufficient budget remaining
73
+ - [ ] Remaining context can carry the next step, or an explicit handoff will be written
74
74
 
75
75
  ---
76
76
 
@@ -104,27 +104,18 @@ Only expand to full cooldown when hitting a mandatory gate.
104
104
 
105
105
  ## Timeout & Retry Enforcement
106
106
 
107
- When a delegated agent does not respond in time, enforce the timeout policy from routing.yml.
108
-
109
- ### Behavior by Agent Role
110
-
111
- | Agent Role | Timeout | Retry Policy | Fallback | Timeout Parcial? |
112
- |------------|---------|-------------|----------|------------------|
113
- | Explorer (apollo) | 60s | 2 retries, exponential backoff | athena | ✅ Yes — partial results OK |
114
- | Implementer (hermes, aphrodite) | 180s | 3 retries, exponential backoff | talos | ❌ No — must produce artifact |
115
- | Reviewer (themis) | 120s | 2 retries, exponential backoff | zeus | ❌ No — must produce verdict |
116
- | Infrastructure (prometheus) | 300s | 2 retries, exponential backoff | hermes | ❌ No |
117
- | Hotfix (talos) | 30s | 1 retry, no backoff | hermes | ✅ Yes — one-liner fix OK |
107
+ When a delegated agent does not respond in time, enforce the timeout policy defined in
108
+ `src/instructions/zeus-timeout-retry.instructions.md`.
118
109
 
110
+ **That file is the single authority** for per-role timeouts, retry counts, fallback chains, and
111
+ whether partial results are acceptable. It is force-fed into every session via `AGENTS.md`; this
112
+ skill is opt-in. Do not restate those values here — a copy in this skill is precisely what drifted
113
+ out of sync with them. If a dispatch times out, read the role's row from the instruction file.
119
114
 
120
115
  ### Timeout Parcial (Partial Results)
121
116
 
122
- Timeout parcial is **only** allowed for read-only, independent agents:
123
- - ✅ @apollo (codebase search) — can return partial file list
124
- - ✅ @gaia (literature review) — can return partial findings
125
- - ✅ @talos (hotfix confirmation) — can return without fix if timeout
126
- - ❌ Never for implementers (@hermes, @aphrodite, @demeter) — must complete or fail
127
- - ❌ Never for reviewers (@themis) — must produce a verdict
117
+ Whether an agent may return partial results is the **Partial Results OK?** column in that table.
118
+ Only agents marked ✅ there may return partial — never implementers, never reviewers.
128
119
 
129
120
  **How to signal timeout parcial:**
130
121
  When dispatching, set the expectation explicitly:
@@ -167,69 +158,28 @@ Ready for Phase 2? Waiting for go-ahead.
167
158
  Should I now run the migration tests? [waiting]
168
159
  ```
169
160
 
170
- ## Session Heartbeat
171
-
172
- Auto-save checkpoint every N turns (configurable, default 5).
173
-
174
- ### Heartbeat File
175
-
176
- `.pantheon/deepwork/<slug>/heartbeat.json`
161
+ ## Session Heartbeat & Checkpoints
177
162
 
178
- ### Heartbeat JSON Format
163
+ Session state is **not** file-based. It lives in **pantheon-persistence** under the
164
+ `checkpoint:<slug>` namespace and is cleaned up automatically by a 4h TTL.
179
165
 
180
- ```json
181
- {
182
- "slug": "task-identifier",
183
- "last_action": "2026-07-16T20:00:00Z",
184
- "turn_count": 10,
185
- "current_phase": 2,
186
- "status": "alive | warning | stalled | paused | completed",
187
- "next_action": "brief description of next planned action"
188
- }
189
- ```
190
-
191
- ### Heartbeat Rules
192
- 1. Update every 5 turns minimum (by default)
193
- 2. Contains NO context — only timestamps and counters
194
- 3. Single file, always overwritten (not versioned)
195
- 4. Used for quick "is this session alive?" checks
196
-
197
- ---
166
+ Write state with `context_save(slug, key, value, session_id=...)` and read it back with
167
+ `context_get(slug, "latest", session_id=...)`. Capture `session_id` from the first `context_save`
168
+ of the session and reuse it for the rest of the session, so the `latest` pointer survives
169
+ compaction.
198
170
 
199
- ## Checkpoint Persistence
200
-
201
- ### Checkpoint File Structure
202
-
203
- Each deepwork task maintains:
204
-
205
- ```
206
- .pantheon/deepwork/<slug>/
207
- ├── PLAN.md # Immutable plan (created at start)
208
- ├── STATUS.md # Human-readable current state (updated every phase)
209
- ├── heartbeat.json # Lightweight ping (updated every N turns)
210
- ├── checkpoint-<N>.json # Full state snapshot (created at phase boundaries)
211
- ├── session.json # Session metadata (created at start, updated on stop)
212
- └── REVIEW.md # Final Themis review (created at end)
213
- ```
214
-
215
- ### Checkpoint JSON Schema
216
-
217
- ```json
218
- {
219
- "slug": "string — unique task identifier",
220
- "phase": "integer — current phase number",
221
- "turn_count": "integer — total turns elapsed",
222
- "timestamp": "string — ISO 8601 timestamp",
223
- "context_hash": "string — first 8 hex chars of SHA-256 of STATUS.md",
224
- "version": "integer — schema version (current: 2)"
225
- }
226
- ```
171
+ ### Heartbeat Check
172
+ - If `context_get(slug, "heartbeat")` returns a checkin older than 300s, log a stall warning and resume
173
+ - Write a heartbeat after every anti-stall recovery action
227
174
 
228
175
  ### Checkpoint Rules
229
- 1. Save at every phase boundary AND before any delegate dispatch
230
- 2. Numbered sequentially (`checkpoint-1.json`, `checkpoint-2.json`, …)
231
- 3. Keep last 10 checkpoints; archive older ones
232
- 4. On resume: read the highest-numbered checkpoint for full state
176
+ 1. Save a checkpoint before **every** delegate dispatch and after every agent returns
177
+ 2. Include current phase, turn count, and remaining tasks
178
+ 3. On resume: `context_get(slug, "latest")` restores the most recent checkpoint
179
+
180
+ > Do not create `heartbeat.json`, `checkpoint-<N>.json`, or `session.json` under
181
+ > `.pantheon/deepwork/`. That file-based mechanism is retired — TTL handles cleanup, and reading
182
+ > a file that nothing writes back returns stale state.
233
183
 
234
184
  ---
235
185
 
@@ -1,255 +0,0 @@
1
- #!/usr/bin/env python3
2
- # ---
3
- # description: LLM judge for skill/agent quality — scores 4 dimensions (0-100) via an OpenAI-compatible endpoint
4
- # timeout: 120
5
- # ---
6
- """LLM judge layer of the plugin-eval certification pipeline.
7
-
8
- Scores a skill/agent directory on 4 dimensions (correctness, maintainability,
9
- security, practicality), each 0-100, via ONE structured prompt to an
10
- OpenAI-compatible chat endpoint. Emits a single JSON document on stdout:
11
-
12
- {name, dimensions: {correctness, maintainability, security, practicality},
13
- overall, notes: {<dimension>: <justification>}}
14
-
15
- Without explicit opt-in it emits a structured skipped result; missing credentials
16
- or failed calls exit 2 with a clear message.
17
-
18
- Env vars:
19
- OPENAI_API_KEY required bearer token
20
- OPENAI_BASE_URL optional endpoint base (default https://api.openai.com/v1)
21
- EVAL_JUDGE_MODEL optional model id (default gpt-4o-mini)
22
-
23
- Stdlib only (urllib.request) — no third-party dependencies.
24
- """
25
-
26
- from __future__ import annotations
27
-
28
- import argparse
29
- import json
30
- import os
31
- import re
32
- import sys
33
- import urllib.error
34
- import urllib.request
35
- from pathlib import Path
36
- from typing import Any
37
- from urllib.parse import urlparse
38
-
39
- DEFAULT_MODEL = "gpt-4o-mini"
40
- DEFAULT_BASE_URL = "https://api.openai.com/v1"
41
- REQUEST_TIMEOUT = 90.0
42
- MAX_CONTENT_CHARS = 12_000
43
- MAX_SUPPORTING_CHARS = 8_000
44
- MAX_SUPPORTING_FILES = 6
45
- DIMENSIONS = ("correctness", "maintainability", "security", "practicality")
46
-
47
- JUDGE_PROMPT = """You are a rigorous quality auditor for AI agent skills and agent definitions used in an agentic coding framework. You decide whether a skill or agent definition is production-quality or AI slop.
48
-
49
- Apply an explicit ANTI-SLOP bias: flag overengineering, unnecessary abstractions, persona proliferation (multiple fake personas/characters), unverifiable claims, hype language, and instructions that sound impressive but are not actionable.
50
-
51
- Score the content on 4 dimensions, each 0-100:
52
- 1. correctness — instructions are accurate and internally consistent; no contradictions; no broken references (files, commands, or agents that do not exist or are misnamed).
53
- 2. maintainability — clear structure, single responsibility, no overengineering, YAGNI-respecting, easy to update.
54
- 3. security — no secrets or credentials; no prompt-injection vectors; no dangerous instructions (e.g. rm -rf, curl | bash, disabling safety checks).
55
- 4. practicality — solves a real problem, actionable, not AI slop or hype; a human would actually use it.
56
-
57
- Return ONLY a JSON object with this exact shape:
58
- {"correctness": <int 0-100>, "maintainability": <int 0-100>, "security": <int 0-100>, "practicality": <int 0-100>, "notes": {"correctness": "<1-2 sentences>", "maintainability": "<1-2 sentences>", "security": "<1-2 sentences>", "practicality": "<1-2 sentences>"}}
59
-
60
- Content to evaluate:
61
- --- BEGIN CONTENT ---
62
- {content}
63
- --- END CONTENT ---
64
- """
65
-
66
- _NAME_RE = re.compile(r"^name:\s*[\"']?([^\"'\n]+?)[\"']?\s*$", re.MULTILINE)
67
-
68
-
69
- class JudgeError(Exception):
70
- """Expected, user-facing judge failure."""
71
-
72
-
73
- def _frontmatter_name(text: str) -> str | None:
74
- """Extract the frontmatter `name` value, or None when absent."""
75
- if not text.startswith("---"):
76
- return None
77
- end = text.find("\n---", 3)
78
- if end == -1:
79
- return None
80
- match = _NAME_RE.search(text[3:end])
81
- return match.group(1).strip() if match else None
82
-
83
-
84
- def _discover_doc(path: Path) -> Path:
85
- """Return the primary doc (SKILL.md or agent .md) for a path."""
86
- if path.is_file():
87
- return path
88
- skill = path / "SKILL.md"
89
- if skill.is_file():
90
- return skill
91
- candidates = sorted(path.glob("*.md"))
92
- for candidate in candidates:
93
- try:
94
- text = candidate.read_text(encoding="utf-8", errors="replace")
95
- except OSError:
96
- continue
97
- if text.startswith("---") and text.find("\n---", 3) != -1:
98
- return candidate
99
- if candidates:
100
- return candidates[0]
101
- raise JudgeError(f"No SKILL.md or .md file found in {path}")
102
-
103
-
104
- def _supporting_files(doc: Path) -> list[Path]:
105
- """Script files next to the doc or in its scripts/ subdir (capped)."""
106
- roots = [doc.parent]
107
- scripts_dir = doc.parent / "scripts"
108
- if scripts_dir.is_dir():
109
- roots.append(scripts_dir)
110
- found: list[Path] = []
111
- for root in roots:
112
- for pattern in ("*.py", "*.sh", "*.js", "*.ts", "*.mjs"):
113
- found.extend(sorted(root.glob(pattern)))
114
- return [p for p in found if p.is_file() and p != doc][:MAX_SUPPORTING_FILES]
115
-
116
-
117
- def _load_content(doc: Path) -> tuple[str, str]:
118
- """Return (name, content) with the doc plus small supporting files."""
119
- try:
120
- text = doc.read_text(encoding="utf-8", errors="replace")
121
- except OSError as exc:
122
- raise JudgeError(f"Cannot read {doc}: {exc}") from exc
123
- name = _frontmatter_name(text) or doc.stem
124
- parts = [f"# {name}\n", text[:MAX_CONTENT_CHARS]]
125
- if len(text) > MAX_CONTENT_CHARS:
126
- parts.append(f"\n[... truncated {len(text) - MAX_CONTENT_CHARS} chars ...]")
127
- support: list[str] = []
128
- total = 0
129
- for sibling in _supporting_files(doc):
130
- try:
131
- data = sibling.read_text(encoding="utf-8", errors="replace")
132
- except OSError:
133
- continue
134
- chunk = f"\n--- supporting file: {sibling.name} ---\n{data[:2000]}"
135
- if total + len(chunk) > MAX_SUPPORTING_CHARS:
136
- break
137
- support.append(chunk)
138
- total += len(chunk)
139
- if support:
140
- parts.append("\n".join(support))
141
- return name, "".join(parts)
142
-
143
-
144
- def _call_llm(prompt: str, allow_external_llm: bool = False) -> str:
145
- """Send one chat completion request via urllib; return the assistant text."""
146
- if not allow_external_llm:
147
- raise JudgeError("External LLM calls require --allow-external-llm or PANTHEON_ALLOW_EXTERNAL_LLM=1")
148
- key = os.getenv("OPENAI_API_KEY")
149
- if not key:
150
- raise JudgeError(
151
- "No LLM available: set OPENAI_API_KEY "
152
- "(optionally OPENAI_BASE_URL and EVAL_JUDGE_MODEL)."
153
- )
154
- base_url = os.getenv("OPENAI_BASE_URL", DEFAULT_BASE_URL).rstrip("/")
155
- parsed_url = urlparse(base_url)
156
- if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc:
157
- raise JudgeError("OPENAI_BASE_URL must use an http or https URL")
158
- endpoint = base_url + "/chat/completions"
159
- model = os.getenv("EVAL_JUDGE_MODEL") or DEFAULT_MODEL
160
- body = {
161
- "model": model,
162
- "messages": [{"role": "user", "content": prompt}],
163
- "temperature": 0.2,
164
- }
165
- request = urllib.request.Request(
166
- endpoint,
167
- data=json.dumps(body).encode("utf-8"),
168
- headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
169
- method="POST",
170
- )
171
- try:
172
- with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT) as response:
173
- payload = json.loads(response.read().decode("utf-8"))
174
- except urllib.error.HTTPError as exc:
175
- detail = exc.read().decode("utf-8", errors="replace")[:200].replace(key, "[redacted]")
176
- raise JudgeError(f"LLM endpoint returned HTTP {exc.code}: {detail}") from exc
177
- except (urllib.error.URLError, TimeoutError, OSError) as exc:
178
- reason = getattr(exc, "reason", exc)
179
- raise JudgeError(f"LLM request failed: {reason}") from exc
180
- except (json.JSONDecodeError, UnicodeDecodeError) as exc:
181
- raise JudgeError(f"LLM returned invalid JSON payload: {exc}") from exc
182
- choices = payload.get("choices") if isinstance(payload, dict) else None
183
- message = choices[0].get("message") if choices else None
184
- text = message.get("content") if isinstance(message, dict) else None
185
- if not isinstance(text, str) or not text.strip():
186
- raise JudgeError("LLM returned an empty response.")
187
- return text.replace(key, "[redacted]")
188
-
189
-
190
- def _parse_json_object(value: str) -> dict[str, Any] | None:
191
- """Parse strict JSON and the fenced JSON commonly returned by models."""
192
- candidates = [value.strip()]
193
- if candidates[0].startswith("```") and candidates[0].endswith("```"):
194
- candidates.append(candidates[0].split("\n", 1)[-1][:-3].strip())
195
- for candidate in candidates:
196
- try:
197
- parsed = json.loads(candidate)
198
- except json.JSONDecodeError:
199
- continue
200
- if isinstance(parsed, dict):
201
- return parsed
202
- return None
203
-
204
-
205
- def _parse_scores(text: str) -> dict[str, Any]:
206
- """Validate the LLM JSON into the report's dimensions/overall/notes."""
207
- parsed = _parse_json_object(text)
208
- if parsed is None:
209
- raise JudgeError("LLM did not return valid JSON.")
210
- dimensions: dict[str, int] = {}
211
- for dim in DIMENSIONS:
212
- raw = parsed.get(dim)
213
- try:
214
- value = int(raw)
215
- except (TypeError, ValueError) as exc:
216
- raise JudgeError(f"LLM returned non-integer score for {dim}: {raw!r}") from exc
217
- if not 0 <= value <= 100:
218
- raise JudgeError(f"LLM returned out-of-range score for {dim}: {value}")
219
- dimensions[dim] = value
220
- raw_notes = parsed.get("notes")
221
- notes = (
222
- {dim: str(raw_notes.get(dim, "")) for dim in DIMENSIONS}
223
- if isinstance(raw_notes, dict)
224
- else {dim: "" for dim in DIMENSIONS}
225
- )
226
- overall = round(sum(dimensions.values()) / len(dimensions), 1)
227
- return {"dimensions": dimensions, "overall": overall, "notes": notes}
228
-
229
-
230
- def main(argv: list[str] | None = None) -> int:
231
- """CLI entry point: `python eval-llm-judge.py <skill-or-agent-dir>`."""
232
- parser = argparse.ArgumentParser(
233
- description="LLM judge for skill/agent quality (plugin-eval layer 2)."
234
- )
235
- parser.add_argument("path", help="Path to a skill/agent directory or doc file")
236
- parser.add_argument("--allow-external-llm", action="store_true", help="Permit sending content to the configured LLM endpoint")
237
- args = parser.parse_args(argv)
238
- allow_external_llm = args.allow_external_llm or os.getenv("PANTHEON_ALLOW_EXTERNAL_LLM") == "1"
239
- try:
240
- doc = _discover_doc(Path(args.path))
241
- name, content = _load_content(doc)
242
- if not allow_external_llm:
243
- print(json.dumps({"name": name, "skipped": True, "reason": "external LLM calls not opted in"}, ensure_ascii=False, indent=2))
244
- return 0
245
- raw = _call_llm(JUDGE_PROMPT.replace("{content}", content), allow_external_llm=True)
246
- result = _parse_scores(raw)
247
- except JudgeError as exc:
248
- print(f"eval-llm-judge: {exc}", file=sys.stderr)
249
- return 2
250
- print(json.dumps({"name": name, **result}, ensure_ascii=False, indent=2))
251
- return 0
252
-
253
-
254
- if __name__ == "__main__":
255
- sys.exit(main())