astron-eval 0.0.1

package/skills/skill-driven-eval/eval-viewer/generate_report.py

@@ -0,0 +1,485 @@
+#!/usr/bin/env python3
+"""Generate a visual comparison report for model evaluation results.
+
+Reads the workspace directory, discovers runs, and generates a self-contained
+HTML page showing the model comparison results.
+
+Usage:
+    python generate_report.py <workspace-path> [--output report.html]
+    python generate_report.py <workspace-path> [--serve] [--port PORT]
+
+No dependencies beyond the Python stdlib are required.
+"""
+
+import argparse
+import base64
+import json
+import mimetypes
+import os
+import signal
+import subprocess
+import sys
+import time
+import webbrowser
+from functools import partial
+from http.server import HTTPServer, BaseHTTPRequestHandler
+from pathlib import Path
+
+# Files to exclude from output listings
+METADATA_FILES = {"transcript.md", "user_notes.md", "metrics.json"}
+
+# Extensions we render as inline text
+TEXT_EXTENSIONS = {
+    ".txt", ".md", ".json", ".csv", ".py", ".js", ".ts", ".tsx", ".jsx",
+    ".yaml", ".yml", ".xml", ".html", ".css", ".sh", ".rb", ".go", ".rs",
+    ".java", ".c", ".cpp", ".h", ".hpp", ".sql", ".r", ".toml",
+}
+
+# Extensions we render as inline images
+IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp"}
+
+# MIME type overrides for common types
+MIME_OVERRIDES = {
+    ".svg": "image/svg+xml",
+    ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+}
+
+
+def get_mime_type(path: Path) -> str:
+    ext = path.suffix.lower()
+    if ext in MIME_OVERRIDES:
+        return MIME_OVERRIDES[ext]
+    mime, _ = mimetypes.guess_type(str(path))
+    return mime or "application/octet-stream"
+
+
+def load_evals_config(workspace: Path) -> dict:
+    """Load evals.json from workspace to get prompts."""
+    evals_path = workspace / "evals.json"
+    if evals_path.exists():
+        try:
+            return json.loads(evals_path.read_text())
+        except (json.JSONDecodeError, OSError):
+            pass
+    return {}
+
+
+def load_mapping(workspace: Path) -> dict:
+    """Load mapping.json to map run IDs to eval info."""
+    mapping_path = workspace / "mapping.json"
+    if mapping_path.exists():
+        try:
+            return json.loads(mapping_path.read_text())
+        except (json.JSONDecodeError, OSError):
+            pass
+    return {}
+
+
+def find_runs(workspace: Path) -> list[dict]:
+    """Recursively find directories that contain an outputs/ subdirectory."""
+    # Load evals and mapping for prompts
+    evals_config = load_evals_config(workspace)
+    mapping = load_mapping(workspace)
+
+    # Build eval_id -> prompt mapping
+    eval_prompts = {}
+    for eval_item in evals_config.get("evals", []):
+        eval_prompts[eval_item.get("id")] = eval_item.get("prompt", "")
+
+    runs: list[dict] = []
+    _find_runs_recursive(workspace, workspace, runs, mapping, eval_prompts)
+    runs.sort(key=lambda r: (r.get("eval_id", float("inf")), r.get("model", ""), r["id"]))
+    return runs
+
+
+def _find_runs_recursive(root: Path, current: Path, runs: list[dict], mapping: dict = None, eval_prompts: dict = None) -> None:
+    if not current.is_dir():
+        return
+
+    outputs_dir = current / "outputs"
+    if outputs_dir.is_dir():
+        run = build_run(root, current, mapping, eval_prompts)
+        if run:
+            runs.append(run)
+        return
+
+    skip = {"node_modules", ".git", "__pycache__", "skill", "inputs"}
+    for child in sorted(current.iterdir()):
+        if child.is_dir() and child.name not in skip:
+            _find_runs_recursive(root, child, runs, mapping, eval_prompts)
+
+
+def build_run(root: Path, run_dir: Path, mapping: dict = None, eval_prompts: dict = None) -> dict | None:
+    """Build a run dict with prompt, outputs, and grading data."""
+    prompt = ""
+    eval_id = None
+    eval_name = None
+    model = run_dir.name  # Model name is directory name
+    run_id = run_dir.name
+
+    # First try mapping.json for run info (blind evaluation mode)
+    if mapping and run_id in mapping:
+        run_info = mapping[run_id]
+        model = run_info.get("model", model)
+        eval_id = run_info.get("eval_id")
+        eval_name = run_info.get("eval_name")
+        # Get prompt from eval_prompts using eval_id
+        if eval_prompts and eval_id in eval_prompts:
+            prompt = eval_prompts[eval_id]
+
+    # Fallback: Try eval_metadata.json from parent
+    if not prompt:
+        metadata_path = run_dir.parent / "eval_metadata.json"
+        if metadata_path.exists():
+            try:
+                metadata = json.loads(metadata_path.read_text())
+                prompt = metadata.get("prompt", "")
+                eval_id = metadata.get("eval_id")
+            except (json.JSONDecodeError, OSError):
+                pass
+
+    # Fallback: Try timing.json for model info
+    if model == run_dir.name:  # Still using directory name
+        timing_path = run_dir / "timing.json"
+        if timing_path.exists():
+            try:
+                timing = json.loads(timing_path.read_text())
+                if timing.get("model"):
+                    model = timing["model"]
+            except (json.JSONDecodeError, OSError):
+                pass
+
+    if not prompt:
+        prompt = "(No prompt found)"
+
+    run_id = str(run_dir.relative_to(root)).replace("/", "-").replace("\\", "-")
+
+    # Define timing_path early (used in multiple places)
+    timing_path = run_dir / "timing.json"
+
+    # Collect output files
+    outputs_dir = run_dir / "outputs"
+    output_files: list[dict] = []
+    if outputs_dir.is_dir():
+        for f in sorted(outputs_dir.iterdir()):
+            if f.is_file() and f.name not in METADATA_FILES:
+                output_files.append(embed_file(f))
+
+    # Load grading if present
+    grading = None
+    grading_path = run_dir / "grading.json"
+    if grading_path.exists():
+        try:
+            grading = json.loads(grading_path.read_text())
+        except (json.JSONDecodeError, OSError):
+            pass
+
+    # Load timing if present
+    timing = None
+    if timing_path.exists():
+        try:
+            timing = json.loads(timing_path.read_text())
+        except (json.JSONDecodeError, OSError):
+            pass
+
+    return {
+        "id": run_id,
+        "prompt": prompt,
+        "eval_id": eval_id,
+        "eval_name": eval_name,
+        "model": model,
+        "outputs": output_files,
+        "grading": grading,
+        "timing": timing,
+    }
+
+
+def embed_file(path: Path) -> dict:
+    """Read a file and return an embedded representation."""
+    ext = path.suffix.lower()
+    mime = get_mime_type(path)
+
+    if ext in TEXT_EXTENSIONS:
+        try:
+            content = path.read_text(errors="replace")
+        except OSError:
+            content = "(Error reading file)"
+        return {
+            "name": path.name,
+            "type": "text",
+            "content": content,
+        }
+    elif ext in IMAGE_EXTENSIONS:
+        try:
+            raw = path.read_bytes()
+            b64 = base64.b64encode(raw).decode("ascii")
+        except OSError:
+            return {"name": path.name, "type": "error", "content": "(Error reading file)"}
+        return {
+            "name": path.name,
+            "type": "image",
+            "mime": mime,
+            "data_uri": f"data:{mime};base64,{b64}",
+        }
+    else:
+        # Binary / unknown — base64 download link
+        try:
+            raw = path.read_bytes()
+            b64 = base64.b64encode(raw).decode("ascii")
+        except OSError:
+            return {"name": path.name, "type": "error", "content": "(Error reading file)"}
+        return {
+            "name": path.name,
+            "type": "binary",
+            "mime": mime,
+            "data_uri": f"data:{mime};base64,{b64}",
+        }
+
+
+def load_benchmark(workspace: Path) -> dict | None:
+    """Load benchmark.json from workspace."""
+    benchmark_path = workspace / "benchmark.json"
+    if benchmark_path.exists():
+        try:
+            return json.loads(benchmark_path.read_text())
+        except (json.JSONDecodeError, OSError):
+            pass
+    return None
+
+
+def generate_html(runs: list[dict], benchmark: dict | None, skill_name: str) -> str:
+    """Generate the complete standalone HTML page with embedded data."""
+    template_path = Path(__file__).parent / "viewer.html"
+    if template_path.exists():
+        template = template_path.read_text()
+    else:
+        template = get_default_template()
+
+    embedded = {
+        "skill_name": skill_name,
+        "runs": runs,
+        "benchmark": benchmark,
+    }
+    # Use indent=2 to avoid extremely long lines that break VSCode tokenization
+    # and ensure ensure_ascii=False for proper Unicode character handling
+    data_json = json.dumps(embedded, indent=2, ensure_ascii=False)
+
+    # CRITICAL: Escape </script> tags in the JSON to prevent browsers from
+    # incorrectly terminating the script block when the data contains HTML examples
+    # Use <\/script> which JavaScript will parse correctly but browsers won't see as a tag
+    data_json = data_json.replace("</script>", "<\\/script>")
+
+    return template.replace("/*__EMBEDDED_DATA__*/", f"const EMBEDDED_DATA = {data_json};")
+
+
+def get_default_template() -> str:
+    """Return a minimal HTML template if viewer.html doesn't exist."""
+    return '''<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Model Comparison Report</title>
+  <style>
+    * { box-sizing: border-box; margin: 0; padding: 0; }
+    body { font-family: system-ui, -apple-system, sans-serif; background: #f5f5f5; padding: 2rem; }
+    h1 { margin-bottom: 1rem; }
+    .card { background: white; border-radius: 8px; padding: 1.5rem; margin-bottom: 1rem; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }
+    .model-badge { display: inline-block; padding: 0.25rem 0.75rem; border-radius: 9999px; font-size: 0.75rem; font-weight: 600; margin-left: 0.5rem; }
+    .model-opus { background: #e3f2fd; color: #1565c0; }
+    .model-sonnet { background: #fff3e0; color: #e65100; }
+    .model-haiku { background: #e8f5e9; color: #2e7d32; }
+    pre { background: #f5f5f5; padding: 1rem; border-radius: 4px; overflow-x: auto; font-size: 0.875rem; }
+    table { width: 100%; border-collapse: collapse; }
+    th, td { padding: 0.75rem; text-align: left; border-bottom: 1px solid #eee; }
+    th { background: #f5f5f5; font-weight: 600; }
+    .pass { color: #2e7d32; }
+    .fail { color: #c62828; }
+  </style>
+</head>
+<body>
+  <h1>Model Comparison: <span id="skill-name"></span></h1>
+  <div id="content"></div>
+  <script>
+    /*__EMBEDDED_DATA__*/
+
+    document.getElementById("skill-name").textContent = EMBEDDED_DATA.skill_name;
+
+    const content = document.getElementById("content");
+
+    // Render benchmark summary if available
+    if (EMBEDDED_DATA.benchmark) {
+      const b = EMBEDDED_DATA.benchmark;
+      const summary = b.model_summary || {};
+      const models = Object.keys(summary);
+
+      let html = '<div class="card"><h2>Summary</h2><table><thead><tr><th>Metric</th>';
+      for (const m of models) {
+        html += `<th>${m}</th>`;
+      }
+      html += '</tr></thead><tbody>';
+
+      html += '<tr><td>Pass Rate</td>';
+      for (const m of models) {
+        const pr = summary[m].pass_rate || {};
+        html += `<td>${(pr.mean * 100).toFixed(0)}% ± ${(pr.stddev * 100).toFixed(0)}%</td>`;
+      }
+      html += '</tr>';
+
+      html += '<tr><td>Time (s)</td>';
+      for (const m of models) {
+        const t = summary[m].time_seconds || {};
+        html += `<td>${t.mean.toFixed(1)}s</td>`;
+      }
+      html += '</tr>';
+
+      html += '<tr><td>Tokens</td>';
+      for (const m of models) {
+        const tk = summary[m].tokens || {};
+        html += `<td>${tk.mean.toFixed(0)}</td>`;
+      }
+      html += '</tr></tbody></table></div>';
+
+      content.innerHTML += html;
+    }
+
+    // Render runs
+    for (const run of EMBEDDED_DATA.runs) {
+      let html = `<div class="card"><h3>Eval ${run.eval_id}: ${run.model}</h3>`;
+      html += `<p><strong>Prompt:</strong> ${run.prompt}</p>`;
+
+      if (run.grading) {
+        const g = run.grading;
+        html += `<p><strong>Pass Rate:</strong> <span class="${g.summary.pass_rate >= 0.7 ? 'pass' : 'fail'}">${(g.summary.pass_rate * 100).toFixed(0)}%</span></p>`;
+
+        if (g.expectations && g.expectations.length > 0) {
+          html += '<h4>Assertions</h4><ul>';
+          for (const e of g.expectations) {
+            html += `<li class="${e.passed ? 'pass' : 'fail'}">${e.passed ? '✓' : '✗'} ${e.text}</li>`;
+          }
+          html += '</ul>';
+        }
+      }
+
+      html += '</div>';
+      content.innerHTML += html;
+    }
+  </script>
+</body>
+</html>'''
+
+
+class ReportHandler(BaseHTTPRequestHandler):
+    """Serves the comparison HTML report."""
+
+    def __init__(
+        self,
+        workspace: Path,
+        skill_name: str,
+        benchmark: dict | None,
+        *args,
+        **kwargs,
+    ):
+        self.workspace = workspace
+        self.skill_name = skill_name
+        self.benchmark = benchmark
+        super().__init__(*args, **kwargs)
+
+    def do_GET(self) -> None:
+        if self.path == "/" or self.path == "/index.html":
+            runs = find_runs(self.workspace)
+            html = generate_html(runs, self.benchmark, self.skill_name)
+            content = html.encode("utf-8")
+            self.send_response(200)
+            self.send_header("Content-Type", "text/html; charset=utf-8")
+            self.send_header("Content-Length", str(len(content)))
+            self.end_headers()
+            self.wfile.write(content)
+        else:
+            self.send_error(404)
+
+    def log_message(self, format: str, *args: object) -> None:
+        pass
+
+
+def _kill_port(port: int) -> None:
+    """Kill any process listening on the given port."""
+    try:
+        result = subprocess.run(
+            ["lsof", "-ti", f":{port}"],
+            capture_output=True, text=True, timeout=5,
+        )
+        for pid_str in result.stdout.strip().split("\n"):
+            if pid_str.strip():
+                try:
+                    os.kill(int(pid_str.strip()), signal.SIGTERM)
+                except (ProcessLookupError, ValueError):
+                    pass
+        if result.stdout.strip():
+            time.sleep(0.5)
+    except subprocess.TimeoutExpired:
+        pass
+    except FileNotFoundError:
+        pass
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Generate model comparison report")
+    parser.add_argument("workspace", type=Path, help="Path to workspace directory")
+    parser.add_argument("--port", "-p", type=int, default=3118, help="Server port (default: 3118)")
+    parser.add_argument("--output", "-o", type=Path, default=None, help="Output HTML file path")
+    parser.add_argument("--serve", "-s", action="store_true", help="Serve report via HTTP")
+    parser.add_argument("--name", "-n", type=str, default=None, help="Skill name for header")
+    args = parser.parse_args()
+
+    workspace = args.workspace.resolve()
+    if not workspace.is_dir():
+        print(f"Error: {workspace} is not a directory", file=sys.stderr)
+        sys.exit(1)
+
+    runs = find_runs(workspace)
+    benchmark = load_benchmark(workspace)
+    skill_name = args.name or workspace.name.replace("-eval-workspace", "")
+
+    if args.output:
+        html = generate_html(runs, benchmark, skill_name)
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(html)
+        print(f"\n  Report written to: {args.output}\n")
+        sys.exit(0)
+
+    if args.serve:
+        port = args.port
+        _kill_port(port)
+        handler = partial(ReportHandler, workspace, skill_name, benchmark)
+        try:
+            server = HTTPServer(("127.0.0.1", port), handler)
+        except OSError:
+            server = HTTPServer(("127.0.0.1", 0), handler)
+            port = server.server_address[1]
+
+        url = f"http://localhost:{port}"
+        print(f"\n  Model Comparison Report")
+        print(f"  ─────────────────────────────────")
+        print(f"  URL:       {url}")
+        print(f"  Workspace: {workspace}")
+        print(f"\n  Press Ctrl+C to stop.\n")
+
+        webbrowser.open(url)
+
+        try:
+            server.serve_forever()
+        except KeyboardInterrupt:
+            print("\nStopped.")
+            server.server_close()
+    else:
+        # Default: write to workspace/report.html
+        output = workspace / "report.html"
+        html = generate_html(runs, benchmark, skill_name)
+        output.write_text(html)
+        print(f"\n  Report written to: {output}\n")
+
+
+if __name__ == "__main__":
+    main()