pantheon-opencode 1.5.2 → 1.6.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/.pantheon/code-mode/README.md +4 -2
  2. package/.pantheon/code-mode/checkpoint_session.py +138 -102
  3. package/.pantheon/code-mode/compress-inline.py +102 -57
  4. package/.pantheon/code-mode/manifest.json +2 -8
  5. package/AGENTS.global.md +182 -0
  6. package/AGENTS.md +10 -891
  7. package/CHANGELOG.md +343 -0
  8. package/README.md +577 -79
  9. package/README.pt-BR.md +345 -60
  10. package/bin/pantheon-init.mjs +26 -3
  11. package/commands/pantheon-audit.md +7 -1
  12. package/commands/pantheon-consolidate.md +1 -1
  13. package/commands/pantheon-deepwork.md +14 -3
  14. package/commands/pantheon-focus.md +17 -0
  15. package/commands/pantheon-optimize.md +2 -2
  16. package/commands/pantheon-vision.md +61 -12
  17. package/package.json +30 -11
  18. package/plugin.json +2 -2
  19. package/pyproject.toml +11 -16
  20. package/scripts/build-agents-md.mjs +376 -58
  21. package/scripts/changelog-merge.mjs +161 -0
  22. package/scripts/check-plugin-v2-coverage.mjs +213 -0
  23. package/scripts/ci-validate-yaml.py +7 -5
  24. package/scripts/{code_mode_server.py → code_mode.py} +119 -27
  25. package/scripts/cost.mjs +244 -41
  26. package/scripts/doctor.mjs +236 -29
  27. package/scripts/install/agents-md.mjs +4 -4
  28. package/scripts/install/cli-ui.mjs +1 -1
  29. package/scripts/install/config-migration.mjs +319 -6
  30. package/scripts/install/health-check.mjs +26 -37
  31. package/scripts/install/interactive.mjs +7 -7
  32. package/scripts/install/migrate.mjs +1 -1
  33. package/scripts/install/opencode-version.mjs +246 -4
  34. package/scripts/install/opencode.mjs +627 -179
  35. package/scripts/install/plugin.mjs +43 -9
  36. package/scripts/install/shared.mjs +120 -12
  37. package/scripts/install/vision-prompt.mjs +161 -0
  38. package/scripts/install-mcp.mjs +28 -9
  39. package/scripts/{mcp_persistence_server.py → mcp_persistence.py} +392 -78
  40. package/{src/mcp/mcp_resources_server.py → scripts/mcp_resources.py} +2 -2
  41. package/scripts/{memory_mcp_server.py → memory_mcp.py} +372 -198
  42. package/scripts/postinstall.mjs +0 -7
  43. package/scripts/probe-context-rehydrate.mjs +36 -13
  44. package/scripts/probe-pantheon-cost.mjs +71 -15
  45. package/scripts/prune.mjs +2 -2
  46. package/scripts/run-node-tests.mjs +189 -0
  47. package/scripts/sync-artifacts.mjs +26 -13
  48. package/scripts/sync-tui.mjs +24 -3
  49. package/scripts/test-opencode-v2-sandbox.sh +1913 -0
  50. package/scripts/validate-routing.mjs +43 -3
  51. package/scripts/validate-sandbox-install.mjs +138 -0
  52. package/scripts/versioning.mjs +13 -1
  53. package/scripts/vision-status.mjs +682 -0
  54. package/skills-lock.json +15 -16
  55. package/src/agents/aphrodite.md +20 -37
  56. package/src/agents/apollo.md +7 -3
  57. package/src/agents/athena.md +63 -35
  58. package/src/agents/demeter.md +8 -28
  59. package/src/agents/gaia.md +1 -1
  60. package/src/agents/hephaestus.md +10 -21
  61. package/src/agents/hermes.md +15 -38
  62. package/src/agents/iris.md +1 -1
  63. package/src/agents/mnemosyne.md +23 -60
  64. package/src/agents/nyx.md +1 -1
  65. package/src/agents/prometheus.md +16 -20
  66. package/src/agents/talos.md +10 -2
  67. package/src/agents/themis.md +5 -5
  68. package/src/agents/zeus.md +99 -130
  69. package/src/instructions/agent-return-format.instructions.md +18 -3
  70. package/src/instructions/backend-standards.instructions.md +2 -0
  71. package/src/instructions/frontend-standards.instructions.md +3 -1
  72. package/src/instructions/memory-protocol.instructions.md +11 -110
  73. package/src/instructions/zeus-anti-stall.instructions.md +14 -61
  74. package/src/instructions/zeus-communication-rules.instructions.md +0 -1
  75. package/src/instructions/zeus-council-synthesis.instructions.md +32 -260
  76. package/src/instructions/zeus-memory-operations.instructions.md +26 -0
  77. package/src/instructions/zeus-timeout-retry.instructions.md +40 -87
  78. package/src/mcp/{code_mode_server.py → code_mode.py} +119 -27
  79. package/src/mcp/eval_store.py +2 -2
  80. package/src/mcp/mcp_codemap_module.py +559 -129
  81. package/src/mcp/{mcp_persistence_server.py → mcp_persistence.py} +392 -78
  82. package/{scripts/mcp_resources_server.py → src/mcp/mcp_resources.py} +2 -2
  83. package/src/mcp/{memory_mcp_server.py → memory_mcp.py} +373 -250
  84. package/src/mcp/{pantheon_vision_server.py → pantheon_vision.py} +396 -40
  85. package/src/mcp/requirements-mcp.txt +10 -6
  86. package/src/mcp/requirements-vision.txt +4 -2
  87. package/src/pantheon/cost-command.ts +399 -101
  88. package/src/pantheon/delegation-enforce.ts +10 -1
  89. package/src/pantheon/model-command.ts +18 -7
  90. package/src/pantheon/presets.d.mts +0 -8
  91. package/src/pantheon/presets.mjs +5 -32
  92. package/src/pantheon/todo-enforcer.ts +1 -1
  93. package/src/pantheon/v2-bounded-read.ts +640 -0
  94. package/src/pantheon/v2-delegation-enforce.ts +214 -0
  95. package/src/pantheon/v2-execute-after.ts +186 -0
  96. package/src/pantheon/v2-tools.ts +429 -0
  97. package/src/pantheon/v2-unsupported.d.mts +11 -0
  98. package/src/pantheon/v2-unsupported.mjs +90 -0
  99. package/src/pantheon/v2-vision.ts +622 -0
  100. package/src/pantheon/vision-local.ts +531 -0
  101. package/src/pantheon/vision.ts +1161 -116
  102. package/src/plugin-v2.ts +1244 -288
  103. package/src/plugin.ts +12 -9
  104. package/src/plugins/tui/dist/server.d.ts +1 -2
  105. package/src/plugins/tui/dist/server.d.ts.map +1 -1
  106. package/src/plugins/tui/dist/tui-v2.d.ts +154 -0
  107. package/src/plugins/tui/dist/tui-v2.d.ts.map +1 -0
  108. package/src/plugins/tui/dist/tui-v2.js +659 -0
  109. package/src/plugins/tui/dist/tui-v2.js.map +1 -0
  110. package/src/plugins/tui/dist/tui.d.ts +85 -82
  111. package/src/plugins/tui/dist/tui.d.ts.map +1 -1
  112. package/src/plugins/tui/dist/tui.js +72 -59
  113. package/src/plugins/tui/index-v2.ts +6 -0
  114. package/src/plugins/tui/package-lock.json +5504 -1341
  115. package/src/plugins/tui/package.json +25 -15
  116. package/src/plugins/tui/src/index-v2.tsx +734 -0
  117. package/src/plugins/tui/src/index.tsx +106 -85
  118. package/src/plugins/tui/src/opencode-v2.d.ts +13 -0
  119. package/src/plugins/tui/tsconfig.build.json +5 -0
  120. package/src/plugins/tui/tsconfig.typecheck.json +5 -0
  121. package/src/plugins/tui/tsdown.config.ts +22 -2
  122. package/src/plugins/tui/tui-v2.ts +2 -0
  123. package/src/plugins/tui/tui.ts +3 -0
  124. package/src/routing.yml +11 -22
  125. package/src/skills/README.md +34 -39
  126. package/src/skills/agent-coordination/SKILL.md +7 -2
  127. package/src/skills/artifact-management/SKILL.md +12 -5
  128. package/src/skills/auto-continue/SKILL.md +54 -95
  129. package/src/skills/code-review-checklist/SKILL.md +5 -5
  130. package/src/skills/context-compression/SKILL.md +26 -879
  131. package/src/skills/session-goal/SKILL.md +14 -5
  132. package/src/skills/spec-driven-development/SKILL.md +3 -1
  133. package/src/skills/tdd-with-agents/SKILL.md +18 -10
  134. package/.pantheon/code-mode/eval-llm-judge.py +0 -255
  135. package/.pantheon/code-mode/eval-monte-carlo.py +0 -368
  136. package/.pantheon/code-mode/eval-run.py +0 -197
  137. package/.pantheon/code-mode/eval-static.py +0 -321
  138. package/.pantheon/code-mode/session-end-save.py +0 -186
  139. package/.pantheon/code-mode/session-end-save.sh +0 -18
  140. package/opencode.json +0 -58
  141. package/scripts/recover-zenodo-deposition.mjs +0 -146
  142. package/scripts/test-opencode-v1-v2-sandbox.sh +0 -862
  143. package/scripts/validate-zenodo-release.mjs +0 -154
  144. package/scripts/zenodo-artifact.mjs +0 -94
  145. package/scripts/zenodo-release-state.mjs +0 -53
  146. package/src/skills/orchestration-workflow/SKILL.md +0 -311
@@ -14,10 +14,10 @@ Use this skill to pin a session objective at the start of a long or complex sess
14
14
 
15
15
  ## Usage
16
16
 
17
- Invoke via the `/focus` command:
17
+ Invoke via the `/pantheon-focus` command (a conversation-only goal anchor):
18
18
 
19
19
  ```
20
- /focus Implement JWT authentication with refresh token rotation
20
+ /pantheon-focus Implement JWT authentication with refresh token rotation
21
21
  ```
22
22
 
23
23
  Or state it explicitly at the start of a session:
@@ -70,9 +70,18 @@ Before each major delegation or decision, run this quick check:
70
70
 
71
71
  ---
72
72
 
73
- ## Cross-Session Persistence
73
+ ## Cross-Session Continuity
74
74
 
75
- For goals that span multiple sessions, write the goal to `/memories/session/`:
75
+ V2 does not provide durable goal tools. `/pantheon-focus` only anchors the current
76
+ conversation and must not be described as surviving compaction or a new
77
+ session. For multi-phase work that needs persistent checkpoints, use
78
+ `/pantheon-deepwork`; otherwise ask before writing a goal to a project file.
79
+
80
+ In V1, use the implemented goal tools when they are available and explicitly
81
+ authorized. Do not infer that their presence in one host generation means they
82
+ exist in V2.
83
+
84
+ Historical path example (not a V2 runtime command):
76
85
 
77
86
  ```
78
87
  /memories/session/current-goal.md
@@ -84,7 +93,7 @@ Status: Phase 2 of 3 (backend done, frontend in progress)
84
93
  Next: POST /auth/refresh endpoint
85
94
  ```
86
95
 
87
- This ensures the goal survives context compaction and session restarts.
96
+ This file-based approach requires explicit authorization and is not automatic.
88
97
 
89
98
  ---
90
99
 
@@ -12,7 +12,9 @@ description: "Define requirements via spec-first before any code — PRD, edge c
12
12
  2. **PRD** — Escreva um documento de requisitos: objetivo, personas, fluxos, regras de negócio, critérios de aceitação
13
13
  3. **Edge Cases** — Mapeie entradas inválidas, concorrência, falhas de API, estados vazios
14
14
  4. **Technical Spec** — Decisões de arquitetura: componentes, dados, API contracts, deploy
15
- 5. **Approval** — Apresente ao usuário antes de codar. "É isso mesmo?"
15
+ 5. **Approval** — Apresente o plano completo ao usuário antes de codar. Se o
16
+ planejamento veio de um subagente, devolva o plano integral ao orquestrador
17
+ para que ele o exiba na conversa principal antes de pedir aprovação.
16
18
 
17
19
  ## Anti-patterns
18
20
  - ❌ Pular direto pra implementação
@@ -8,13 +8,21 @@ alwaysApply: false
8
8
 
9
9
  # TDD with Agents
10
10
 
11
- Enforce RED → GREEN → REFACTOR cycle across all implementation agents. Includes advanced testing patterns for E2E, load, mutation, and contract testing.
11
+ Use RED → GREEN → REFACTOR for behavior-changing implementation where a meaningful test can express the change. Keep verification proportional to the changed surface and risk; advanced testing patterns are reserved for changes that warrant them.
12
12
 
13
13
  ---
14
14
 
15
15
  ## Core Principle
16
16
 
17
- > **Write the test FIRST. Watch it fail. Write minimal code to pass. Refactor with confidence.**
17
+ > **For testable behavior changes, write the focused test first, observe RED, implement minimally, then refactor.**
18
+
19
+ ## Proportional TDD
20
+
21
+ - Use the complete RED → GREEN → REFACTOR cycle for new or changed behavior when a focused test is practical. Include a focused regression check for bug fixes.
22
+ - For typo, formatting, documentation-only, or otherwise non-behavioral edits, do not invent a test; inspect the diff and run the narrowest relevant validation.
23
+ - Run targeted tests for a small change. Run broader suites when the changed surface, integration boundaries, or risk make them useful; do not run the whole suite solely because one line changed.
24
+ - If a relevant existing test already captures the behavior, use it instead of adding a duplicate. If no automated check can express the change, report the manual or static verification performed.
25
+ - Auth, security, payment, data-integrity, and schema/migration changes retain focused RED/GREEN tests, required regression coverage, and Themis review. Do not weaken a project-specific required safety gate.
18
26
 
19
27
  ---
20
28
 
@@ -112,12 +120,12 @@ def test_provider_satisfies_consumer_contract():
112
120
  pass
113
121
  ```
114
122
 
115
- ### Mutation Testing
123
+ ### Mutation Testing (high-risk/critical logic only)
116
124
  - Mutate source code (change `>` to `<`, remove conditions)
117
125
  - Run tests → should fail (mutation killed)
118
126
  - If tests pass → test is weak; improve it
119
127
 
120
- ### Visual Regression
128
+ ### Visual Regression (visual behavior only)
121
129
  ```typescript
122
130
  test('homepage looks the same', async ({ page }) => {
123
131
  await page.goto('/')
@@ -129,8 +137,8 @@ test('homepage looks the same', async ({ page }) => {
129
137
 
130
138
  ## Coverage Rules
131
139
 
132
- - **Minimum: 80%** for all code
133
- - **Critical paths: 100%** (auth, payments, data integrity)
140
+ - Follow a repository's explicit coverage threshold when the changed code is in its measurement scope; do not manufacture a coverage target for docs, formatting, or a tiny isolated edit.
141
+ - **Critical paths:** preserve project-required coverage and tests for auth, payments, and data integrity.
134
142
  - **No snapshot testing** — test behavior, not output shape
135
143
  - **Test edge cases**: empty input, null, boundary values, errors
136
144
 
@@ -143,20 +151,20 @@ test('homepage looks the same', async ({ page }) => {
143
151
  | **Hermes** | Unit + integration for FastAPI endpoints, services, middleware |
144
152
  | **Aphrodite** | Component behavior tests with React Testing Library |
145
153
  | **Demeter** | Migration upgrade/downgrade, query correctness |
146
- | **Themis** | Verifies coverage ≥80%, edge cases tested, error conditions |
154
+ | **Themis** | Reviews the changed surface, relevant edge/error cases, and applicable repository coverage requirements; always review sensitive/material changes |
147
155
 
148
156
  ---
149
157
 
150
158
  ## Workflow
151
159
 
152
160
  ```
153
- 1. Zeus/Athena defines feature with testable acceptance criteria
161
+ 1. Use a plan/spec only when scope, risk, or ambiguity warrants it; a bounded fix may proceed directly
154
162
  2. Hermes writes failing test (RED)
155
163
  3. Hermes implements minimal code (GREEN)
156
164
  4. Hermes refactors (REFACTOR)
157
165
  5. Aphrodite writes frontend tests in parallel
158
- 6. Themis reviews: coverage, edge cases, correctness
159
- 7. If coverage <80% → NEEDS_REVISION
166
+ 6. Themis reviews sensitive or material changes, checking changed behavior and applicable quality gates
167
+ 7. Escalate unmet repository or safety-critical test requirements; do not fail a micro-edit against an unrelated global coverage figure
160
168
  ```
161
169
 
162
170
  ---
@@ -1,255 +0,0 @@
1
- #!/usr/bin/env python3
2
- # ---
3
- # description: LLM judge for skill/agent quality — scores 4 dimensions (0-100) via an OpenAI-compatible endpoint
4
- # timeout: 120
5
- # ---
6
- """LLM judge layer of the plugin-eval certification pipeline.
7
-
8
- Scores a skill/agent directory on 4 dimensions (correctness, maintainability,
9
- security, practicality), each 0-100, via ONE structured prompt to an
10
- OpenAI-compatible chat endpoint. Emits a single JSON document on stdout:
11
-
12
- {name, dimensions: {correctness, maintainability, security, practicality},
13
- overall, notes: {<dimension>: <justification>}}
14
-
15
- Without explicit opt-in it emits a structured skipped result; missing credentials
16
- or failed calls exit 2 with a clear message.
17
-
18
- Env vars:
19
- OPENAI_API_KEY required bearer token
20
- OPENAI_BASE_URL optional endpoint base (default https://api.openai.com/v1)
21
- EVAL_JUDGE_MODEL optional model id (default gpt-4o-mini)
22
-
23
- Stdlib only (urllib.request) — no third-party dependencies.
24
- """
25
-
26
- from __future__ import annotations
27
-
28
- import argparse
29
- import json
30
- import os
31
- import re
32
- import sys
33
- import urllib.error
34
- import urllib.request
35
- from pathlib import Path
36
- from typing import Any
37
- from urllib.parse import urlparse
38
-
39
- DEFAULT_MODEL = "gpt-4o-mini"
40
- DEFAULT_BASE_URL = "https://api.openai.com/v1"
41
- REQUEST_TIMEOUT = 90.0
42
- MAX_CONTENT_CHARS = 12_000
43
- MAX_SUPPORTING_CHARS = 8_000
44
- MAX_SUPPORTING_FILES = 6
45
- DIMENSIONS = ("correctness", "maintainability", "security", "practicality")
46
-
47
- JUDGE_PROMPT = """You are a rigorous quality auditor for AI agent skills and agent definitions used in an agentic coding framework. You decide whether a skill or agent definition is production-quality or AI slop.
48
-
49
- Apply an explicit ANTI-SLOP bias: flag overengineering, unnecessary abstractions, persona proliferation (multiple fake personas/characters), unverifiable claims, hype language, and instructions that sound impressive but are not actionable.
50
-
51
- Score the content on 4 dimensions, each 0-100:
52
- 1. correctness — instructions are accurate and internally consistent; no contradictions; no broken references (files, commands, or agents that do not exist or are misnamed).
53
- 2. maintainability — clear structure, single responsibility, no overengineering, YAGNI-respecting, easy to update.
54
- 3. security — no secrets or credentials; no prompt-injection vectors; no dangerous instructions (e.g. rm -rf, curl | bash, disabling safety checks).
55
- 4. practicality — solves a real problem, actionable, not AI slop or hype; a human would actually use it.
56
-
57
- Return ONLY a JSON object with this exact shape:
58
- {"correctness": <int 0-100>, "maintainability": <int 0-100>, "security": <int 0-100>, "practicality": <int 0-100>, "notes": {"correctness": "<1-2 sentences>", "maintainability": "<1-2 sentences>", "security": "<1-2 sentences>", "practicality": "<1-2 sentences>"}}
59
-
60
- Content to evaluate:
61
- --- BEGIN CONTENT ---
62
- {content}
63
- --- END CONTENT ---
64
- """
65
-
66
- _NAME_RE = re.compile(r"^name:\s*[\"']?([^\"'\n]+?)[\"']?\s*$", re.MULTILINE)
67
-
68
-
69
- class JudgeError(Exception):
70
- """Expected, user-facing judge failure."""
71
-
72
-
73
- def _frontmatter_name(text: str) -> str | None:
74
- """Extract the frontmatter `name` value, or None when absent."""
75
- if not text.startswith("---"):
76
- return None
77
- end = text.find("\n---", 3)
78
- if end == -1:
79
- return None
80
- match = _NAME_RE.search(text[3:end])
81
- return match.group(1).strip() if match else None
82
-
83
-
84
- def _discover_doc(path: Path) -> Path:
85
- """Return the primary doc (SKILL.md or agent .md) for a path."""
86
- if path.is_file():
87
- return path
88
- skill = path / "SKILL.md"
89
- if skill.is_file():
90
- return skill
91
- candidates = sorted(path.glob("*.md"))
92
- for candidate in candidates:
93
- try:
94
- text = candidate.read_text(encoding="utf-8", errors="replace")
95
- except OSError:
96
- continue
97
- if text.startswith("---") and text.find("\n---", 3) != -1:
98
- return candidate
99
- if candidates:
100
- return candidates[0]
101
- raise JudgeError(f"No SKILL.md or .md file found in {path}")
102
-
103
-
104
- def _supporting_files(doc: Path) -> list[Path]:
105
- """Script files next to the doc or in its scripts/ subdir (capped)."""
106
- roots = [doc.parent]
107
- scripts_dir = doc.parent / "scripts"
108
- if scripts_dir.is_dir():
109
- roots.append(scripts_dir)
110
- found: list[Path] = []
111
- for root in roots:
112
- for pattern in ("*.py", "*.sh", "*.js", "*.ts", "*.mjs"):
113
- found.extend(sorted(root.glob(pattern)))
114
- return [p for p in found if p.is_file() and p != doc][:MAX_SUPPORTING_FILES]
115
-
116
-
117
- def _load_content(doc: Path) -> tuple[str, str]:
118
- """Return (name, content) with the doc plus small supporting files."""
119
- try:
120
- text = doc.read_text(encoding="utf-8", errors="replace")
121
- except OSError as exc:
122
- raise JudgeError(f"Cannot read {doc}: {exc}") from exc
123
- name = _frontmatter_name(text) or doc.stem
124
- parts = [f"# {name}\n", text[:MAX_CONTENT_CHARS]]
125
- if len(text) > MAX_CONTENT_CHARS:
126
- parts.append(f"\n[... truncated {len(text) - MAX_CONTENT_CHARS} chars ...]")
127
- support: list[str] = []
128
- total = 0
129
- for sibling in _supporting_files(doc):
130
- try:
131
- data = sibling.read_text(encoding="utf-8", errors="replace")
132
- except OSError:
133
- continue
134
- chunk = f"\n--- supporting file: {sibling.name} ---\n{data[:2000]}"
135
- if total + len(chunk) > MAX_SUPPORTING_CHARS:
136
- break
137
- support.append(chunk)
138
- total += len(chunk)
139
- if support:
140
- parts.append("\n".join(support))
141
- return name, "".join(parts)
142
-
143
-
144
- def _call_llm(prompt: str, allow_external_llm: bool = False) -> str:
145
- """Send one chat completion request via urllib; return the assistant text."""
146
- if not allow_external_llm:
147
- raise JudgeError("External LLM calls require --allow-external-llm or PANTHEON_ALLOW_EXTERNAL_LLM=1")
148
- key = os.getenv("OPENAI_API_KEY")
149
- if not key:
150
- raise JudgeError(
151
- "No LLM available: set OPENAI_API_KEY "
152
- "(optionally OPENAI_BASE_URL and EVAL_JUDGE_MODEL)."
153
- )
154
- base_url = os.getenv("OPENAI_BASE_URL", DEFAULT_BASE_URL).rstrip("/")
155
- parsed_url = urlparse(base_url)
156
- if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc:
157
- raise JudgeError("OPENAI_BASE_URL must use an http or https URL")
158
- endpoint = base_url + "/chat/completions"
159
- model = os.getenv("EVAL_JUDGE_MODEL") or DEFAULT_MODEL
160
- body = {
161
- "model": model,
162
- "messages": [{"role": "user", "content": prompt}],
163
- "temperature": 0.2,
164
- }
165
- request = urllib.request.Request(
166
- endpoint,
167
- data=json.dumps(body).encode("utf-8"),
168
- headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
169
- method="POST",
170
- )
171
- try:
172
- with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT) as response:
173
- payload = json.loads(response.read().decode("utf-8"))
174
- except urllib.error.HTTPError as exc:
175
- detail = exc.read().decode("utf-8", errors="replace")[:200].replace(key, "[redacted]")
176
- raise JudgeError(f"LLM endpoint returned HTTP {exc.code}: {detail}") from exc
177
- except (urllib.error.URLError, TimeoutError, OSError) as exc:
178
- reason = getattr(exc, "reason", exc)
179
- raise JudgeError(f"LLM request failed: {reason}") from exc
180
- except (json.JSONDecodeError, UnicodeDecodeError) as exc:
181
- raise JudgeError(f"LLM returned invalid JSON payload: {exc}") from exc
182
- choices = payload.get("choices") if isinstance(payload, dict) else None
183
- message = choices[0].get("message") if choices else None
184
- text = message.get("content") if isinstance(message, dict) else None
185
- if not isinstance(text, str) or not text.strip():
186
- raise JudgeError("LLM returned an empty response.")
187
- return text.replace(key, "[redacted]")
188
-
189
-
190
- def _parse_json_object(value: str) -> dict[str, Any] | None:
191
- """Parse strict JSON and the fenced JSON commonly returned by models."""
192
- candidates = [value.strip()]
193
- if candidates[0].startswith("```") and candidates[0].endswith("```"):
194
- candidates.append(candidates[0].split("\n", 1)[-1][:-3].strip())
195
- for candidate in candidates:
196
- try:
197
- parsed = json.loads(candidate)
198
- except json.JSONDecodeError:
199
- continue
200
- if isinstance(parsed, dict):
201
- return parsed
202
- return None
203
-
204
-
205
- def _parse_scores(text: str) -> dict[str, Any]:
206
- """Validate the LLM JSON into the report's dimensions/overall/notes."""
207
- parsed = _parse_json_object(text)
208
- if parsed is None:
209
- raise JudgeError("LLM did not return valid JSON.")
210
- dimensions: dict[str, int] = {}
211
- for dim in DIMENSIONS:
212
- raw = parsed.get(dim)
213
- try:
214
- value = int(raw)
215
- except (TypeError, ValueError) as exc:
216
- raise JudgeError(f"LLM returned non-integer score for {dim}: {raw!r}") from exc
217
- if not 0 <= value <= 100:
218
- raise JudgeError(f"LLM returned out-of-range score for {dim}: {value}")
219
- dimensions[dim] = value
220
- raw_notes = parsed.get("notes")
221
- notes = (
222
- {dim: str(raw_notes.get(dim, "")) for dim in DIMENSIONS}
223
- if isinstance(raw_notes, dict)
224
- else {dim: "" for dim in DIMENSIONS}
225
- )
226
- overall = round(sum(dimensions.values()) / len(dimensions), 1)
227
- return {"dimensions": dimensions, "overall": overall, "notes": notes}
228
-
229
-
230
- def main(argv: list[str] | None = None) -> int:
231
- """CLI entry point: `python eval-llm-judge.py <skill-or-agent-dir>`."""
232
- parser = argparse.ArgumentParser(
233
- description="LLM judge for skill/agent quality (plugin-eval layer 2)."
234
- )
235
- parser.add_argument("path", help="Path to a skill/agent directory or doc file")
236
- parser.add_argument("--allow-external-llm", action="store_true", help="Permit sending content to the configured LLM endpoint")
237
- args = parser.parse_args(argv)
238
- allow_external_llm = args.allow_external_llm or os.getenv("PANTHEON_ALLOW_EXTERNAL_LLM") == "1"
239
- try:
240
- doc = _discover_doc(Path(args.path))
241
- name, content = _load_content(doc)
242
- if not allow_external_llm:
243
- print(json.dumps({"name": name, "skipped": True, "reason": "external LLM calls not opted in"}, ensure_ascii=False, indent=2))
244
- return 0
245
- raw = _call_llm(JUDGE_PROMPT.replace("{content}", content), allow_external_llm=True)
246
- result = _parse_scores(raw)
247
- except JudgeError as exc:
248
- print(f"eval-llm-judge: {exc}", file=sys.stderr)
249
- return 2
250
- print(json.dumps({"name": name, **result}, ensure_ascii=False, indent=2))
251
- return 0
252
-
253
-
254
- if __name__ == "__main__":
255
- sys.exit(main())