astron-eval 0.0.1

package/skills/skill-driven-eval/scripts/extract_transcript.py

@@ -0,0 +1,294 @@
+#!/usr/bin/env python3
+"""
+Extract and format session transcript into human-readable markdown.
+
+This script reads a Claude Code session transcript (JSONL format) and generates
+a readable markdown document showing the conversation flow:
+- User messages
+- Assistant responses (with thinking blocks)
+- Tool calls and their results
+
+Usage:
+    python extract_transcript.py <session.jsonl> --output <output.md>
+
+The output is suitable for human review but does NOT include model identification,
+maintaining blind evaluation integrity.
+"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from datetime import datetime
+
+
+def format_timestamp(ts: float | None) -> str:
+    """Format Unix timestamp to readable string."""
+    if ts is None:
+        return ""
+    try:
+        return datetime.fromtimestamp(ts).strftime("%H:%M:%S")
+    except (ValueError, TypeError):
+        return ""
+
+
+def format_content_block(block: dict, indent: str = "") -> list[str]:
+    """Format a single content block to markdown lines."""
+    lines = []
+    block_type = block.get("type", "unknown")
+
+    if block_type == "text":
+        text = block.get("text", "")
+        if text.strip():
+            lines.append(f"{indent}{text}")
+
+    elif block_type == "thinking":
+        thinking = block.get("thinking", "")
+        if thinking.strip():
+            lines.append(f"{indent}### Thinking")
+            lines.append(f"{indent}```")
+            lines.append(f"{indent}{thinking}")
+            lines.append(f"{indent}```")
+            lines.append("")
+
+    elif block_type == "tool_use":
+        tool_name = block.get("name", "unknown")
+        tool_id = block.get("id", "")
+        tool_input = block.get("input", {})
+
+        lines.append(f"{indent}### Tool: `{tool_name}`")
+        lines.append(f"{indent}```json")
+        # Format input nicely, truncate if too long
+        input_str = json.dumps(tool_input, indent=2, ensure_ascii=False)
+        if len(input_str) > 2000:
+            input_str = input_str[:2000] + "\n... (truncated)"
+        lines.append(f"{indent}{input_str}")
+        lines.append(f"{indent}```")
+        lines.append("")
+
+    elif block_type == "redacted_thinking":
+        lines.append(f"{indent}*[Redacted thinking block]*")
+        lines.append("")
+
+    return lines
+
+
+def format_tool_result(result: dict, indent: str = "") -> list[str]:
+    """Format a tool result block."""
+    lines = []
+    tool_use_id = result.get("tool_use_id", "")
+    content = result.get("content", "")
+    is_error = result.get("is_error", False)
+
+    header = "### Tool Result"
+    if is_error:
+        header = "### Tool Result (Error)"
+
+    lines.append(f"{indent}{header}")
+    lines.append(f"{indent}```")
+
+    # Handle different content types
+    if isinstance(content, str):
+        # Truncate long results
+        if len(content) > 3000:
+            content = content[:3000] + "\n... (truncated)"
+        lines.append(f"{indent}{content}")
+    elif isinstance(content, list):
+        for item in content:
+            if isinstance(item, dict) and item.get("type") == "text":
+                text = item.get("text", "")
+                if len(text) > 3000:
+                    text = text[:3000] + "\n... (truncated)"
+                lines.append(f"{indent}{text}")
+            elif isinstance(item, dict) and item.get("type") == "image":
+                lines.append(f"{indent}[Image: {item.get('source', {}).get('media_type', 'unknown')}]")
+            else:
+                item_str = str(item)
+                if len(item_str) > 500:
+                    item_str = item_str[:500] + "... (truncated)"
+                lines.append(f"{indent}{item_str}")
+    else:
+        content_str = str(content)
+        if len(content_str) > 3000:
+            content_str = content_str[:3000] + "\n... (truncated)"
+        lines.append(f"{indent}{content_str}")
+
+    lines.append(f"{indent}```")
+    lines.append("")
+
+    return lines
+
+
+def extract_transcript(input_path: Path, output_path: Path, include_metadata: bool = True):
+    """
+    Extract transcript from JSONL session file to markdown.
+
+    Args:
+        input_path: Path to session.jsonl file
+        output_path: Path to output markdown file
+        include_metadata: Whether to include timing and token metadata
+    """
+    if not input_path.exists():
+        print(f"Error: Input file not found: {input_path}", file=sys.stderr)
+        sys.exit(1)
+
+    lines = []
+
+    # Header
+    lines.append("# Session Transcript")
+    lines.append("")
+    lines.append(f"*Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*")
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+
+    # Track message order for tool results
+    pending_tool_results = {}
+    message_count = 0
+    total_tokens = 0
+
+    with open(input_path, 'r', encoding='utf-8') as f:
+        for line_num, line in enumerate(f, 1):
+            line = line.strip()
+            if not line:
+                continue
+
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError as e:
+                lines.append(f"*Error parsing line {line_num}: {e}*")
+                lines.append("")
+                continue
+
+            entry_type = entry.get("type", "")
+
+            # Handle different entry types
+            if entry_type == "summary":
+                # Skip summary entries
+                continue
+
+            # Get role - could be at top level or in message
+            role = entry.get("role", "")
+            if not role and "message" in entry:
+                role = entry.get("message", {}).get("role", "")
+
+            # Get content blocks
+            content = entry.get("content", [])
+            if not content and "message" in entry:
+                content = entry.get("message", {}).get("content", [])
+
+            # Get timestamp
+            timestamp = entry.get("timestamp") or entry.get("message", {}).get("timestamp")
+
+            # Get usage info
+            usage = entry.get("usage") or entry.get("message", {}).get("usage", {})
+            if usage:
+                total_tokens += usage.get("input_tokens", 0) + usage.get("output_tokens", 0)
+
+            if role == "user":
+                message_count += 1
+                ts_str = format_timestamp(timestamp)
+                lines.append(f"## User Message {message_count}")
+                if ts_str and include_metadata:
+                    lines.append(f"*Time: {ts_str}*")
+                lines.append("")
+
+                # Handle content that is a string directly
+                if isinstance(content, str):
+                    lines.append(content)
+                elif isinstance(content, list):
+                    for block in content:
+                        if isinstance(block, dict):
+                            lines.extend(format_content_block(block))
+                        elif isinstance(block, str):
+                            lines.append(block)
+                lines.append("---")
+                lines.append("")
+
+            elif role == "assistant":
+                message_count += 1
+                ts_str = format_timestamp(timestamp)
+                lines.append(f"## Assistant Response {message_count}")
+                if ts_str and include_metadata:
+                    lines.append(f"*Time: {ts_str}*")
+                    if usage:
+                        input_t = usage.get("input_tokens", 0)
+                        output_t = usage.get("output_tokens", 0)
+                        lines.append(f"*Tokens: {input_t} in, {output_t} out*")
+                lines.append("")
+
+                # Handle content that is a string directly
+                if isinstance(content, str):
+                    lines.append(content)
+                elif isinstance(content, list):
+                    for block in content:
+                        if isinstance(block, dict):
+                            block_type = block.get("type", "")
+                            if block_type == "tool_use":
+                                # Store tool call for potential result matching
+                                tool_id = block.get("id", "")
+                                pending_tool_results[tool_id] = message_count
+                            lines.extend(format_content_block(block))
+                        elif isinstance(block, str):
+                            lines.append(block)
+                lines.append("---")
+                lines.append("")
+
+            # Handle tool results in user content
+            if role == "user" and isinstance(content, list):
+                for block in content:
+                    if isinstance(block, dict) and block.get("type") == "tool_result":
+                        lines.extend(format_tool_result(block))
+
+    # Footer with summary
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+    lines.append("## Summary")
+    lines.append("")
+    lines.append(f"- **Total messages**: {message_count}")
+    if include_metadata and total_tokens > 0:
+        lines.append(f"- **Total tokens**: {total_tokens}")
+    lines.append("")
+
+    # Write output
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text("\n".join(lines), encoding='utf-8')
+    print(f"Generated: {output_path}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Extract session transcript to human-readable markdown"
+    )
+    parser.add_argument(
+        "input",
+        type=Path,
+        help="Path to session.jsonl file"
+    )
+    parser.add_argument(
+        "--output", "-o",
+        type=Path,
+        help="Output markdown file path (default: same name with .md extension)"
+    )
+    parser.add_argument(
+        "--no-metadata",
+        action="store_true",
+        help="Exclude timing and token metadata"
+    )
+
+    args = parser.parse_args()
+
+    output_path = args.output
+    if output_path is None:
+        output_path = args.input.with_suffix(".md")
+
+    extract_transcript(
+        args.input,
+        output_path,
+        include_metadata=not args.no_metadata
+    )
+
+
+if __name__ == "__main__":
+    main()

package/skills/skill-driven-eval/scripts/test_aggregate.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python3
+"""Test script for aggregate_results.py.
+
+Creates sample test data with anonymous run IDs and mapping.json,
+then runs the aggregation to verify it works correctly.
+
+Run with: python test_aggregate.py
+"""
+
+import json
+import shutil
+import sys
+from pathlib import Path
+
+
+def create_test_workspace():
+    """Create a test workspace with sample data using anonymous run IDs."""
+    test_dir = Path(__file__).parent.parent / "test-workspace"
+
+    # Clean up if exists
+    if test_dir.exists():
+        shutil.rmtree(test_dir)
+
+    test_dir.mkdir(parents=True)
+
+    # Create evals.json
+    evals = {
+        "target_skill": "pdf",
+        "target_skill_path": "/path/to/pdf-skill",
+        "models_to_compare": ["opus", "sonnet"],
+        "evals": [
+            {
+                "id": 1,
+                "name": "Form Filling",
+                "prompt": "Fill out the PDF form with the following data: name=John Doe, email=john@example.com",
+                "expected_output": "A filled PDF form with all fields populated",
+                "assertions": [
+                    "The output is a PDF file",
+                    "The name field contains 'John Doe'",
+                    "The email field contains 'john@example.com'"
+                ]
+            },
+            {
+                "id": 2,
+                "name": "Text Extraction",
+                "prompt": "Extract all text from the invoice PDF and create a summary",
+                "expected_output": "A text summary of the invoice contents",
+                "assertions": [
+                    "The output contains extracted text",
+                    "The total amount is mentioned",
+                    "The date is correctly extracted"
+                ]
+            }
+        ]
+    }
+
+    with open(test_dir / "evals.json", "w") as f:
+        json.dump(evals, f, indent=2)
+
+    # Mapping from run IDs to models (created by MainAgent, but for testing we create it now)
+    # In real usage, this would be created AFTER grading is complete
+    mapping = {
+        "run-001": {"model": "opus", "eval_id": 1, "eval_name": "Form Filling"},
+        "run-002": {"model": "sonnet", "eval_id": 1, "eval_name": "Form Filling"},
+        "run-003": {"model": "opus", "eval_id": 2, "eval_name": "Text Extraction"},
+        "run-004": {"model": "sonnet", "eval_id": 2, "eval_name": "Text Extraction"}
+    }
+
+    # Create run directories with anonymous IDs
+    runs_data = {
+        "run-001": {
+            "grading": {
+                "run_id": "run-001",
+                "expectations": [
+                    {"text": "The output is a PDF file", "passed": True, "evidence": "Output file is filled_form.pdf"},
+                    {"text": "The name field contains 'John Doe'", "passed": True, "evidence": "Found in field mapping"},
+                    {"text": "The email field contains 'john@example.com'", "passed": True, "evidence": "Verified in output"}
+                ],
+                "summary": {"passed": 3, "failed": 0, "total": 3, "pass_rate": 1.0},
+                "execution_metrics": {"total_tool_calls": 12, "errors_encountered": 0}
+            },
+            "timing": {
+                "run_id": "run-001",
+                "total_tokens": 45000,
+                "duration_ms": 45200,
+                "total_duration_seconds": 45.2
+            }
+        },
+        "run-002": {
+            "grading": {
+                "run_id": "run-002",
+                "expectations": [
+                    {"text": "The output is a PDF file", "passed": True, "evidence": "Output file is filled_form.pdf"},
+                    {"text": "The name field contains 'John Doe'", "passed": True, "evidence": "Found in field mapping"},
+                    {"text": "The email field contains 'john@example.com'", "passed": False, "evidence": "Email was truncated"}
+                ],
+                "summary": {"passed": 2, "failed": 1, "total": 3, "pass_rate": 0.67},
+                "execution_metrics": {"total_tool_calls": 10, "errors_encountered": 0}
+            },
+            "timing": {
+                "run_id": "run-002",
+                "total_tokens": 28000,
+                "duration_ms": 32100,
+                "total_duration_seconds": 32.1
+            }
+        },
+        "run-003": {
+            "grading": {
+                "run_id": "run-003",
+                "expectations": [
+                    {"text": "The output contains extracted text", "passed": True, "evidence": "Text extracted successfully"},
+                    {"text": "The total amount is mentioned", "passed": True, "evidence": "Found '$1,234.56' in output"},
+                    {"text": "The date is correctly extracted", "passed": True, "evidence": "Date: 2024-03-15"}
+                ],
+                "summary": {"passed": 3, "failed": 0, "total": 3, "pass_rate": 1.0},
+                "execution_metrics": {"total_tool_calls": 8, "errors_encountered": 0}
+            },
+            "timing": {
+                "run_id": "run-003",
+                "total_tokens": 38000,
+                "duration_ms": 28500,
+                "total_duration_seconds": 28.5
+            }
+        },
+        "run-004": {
+            "grading": {
+                "run_id": "run-004",
+                "expectations": [
+                    {"text": "The output contains extracted text", "passed": True, "evidence": "Text extracted"},
+                    {"text": "The total amount is mentioned", "passed": False, "evidence": "Amount not found in output"},
+                    {"text": "The date is correctly extracted", "passed": True, "evidence": "Date: 2024-03-15"}
+                ],
+                "summary": {"passed": 2, "failed": 1, "total": 3, "pass_rate": 0.67},
+                "execution_metrics": {"total_tool_calls": 7, "errors_encountered": 0}
+            },
+            "timing": {
+                "run_id": "run-004",
+                "total_tokens": 24000,
+                "duration_ms": 22300,
+                "total_duration_seconds": 22.3
+            }
+        }
+    }
+
+    # Create run directories
+    for run_id, data in runs_data.items():
+        run_dir = test_dir / run_id
+        run_dir.mkdir()
+        (run_dir / "outputs").mkdir()
+
+        # Write grading.json (blind - no model info)
+        with open(run_dir / "grading.json", "w") as f:
+            json.dump(data["grading"], f, indent=2)
+
+        # Write timing.json (blind - no model info)
+        with open(run_dir / "timing.json", "w") as f:
+            json.dump(data["timing"], f, indent=2)
+
+    # Write mapping.json (would be created by MainAgent after grading)
+    # For testing, we create it now
+    with open(test_dir / "mapping.json", "w") as f:
+        json.dump(mapping, f, indent=2)
+
+    print(f"Created test workspace at: {test_dir}")
+    print(f"\nNote: In real usage, mapping.json is created by MainAgent AFTER grading.")
+    print(f"For this test, we create it upfront to verify aggregation.")
+    return test_dir
+
+
+def run_aggregation(test_dir):
+    """Run the aggregation script on the test workspace."""
+    sys.path.insert(0, str(Path(__file__).parent))
+    from aggregate_results import generate_benchmark, generate_markdown, load_mapping
+
+    # Load the mapping
+    mapping = load_mapping(test_dir)
+
+    benchmark = generate_benchmark(test_dir, mapping)
+
+    # Print summary
+    print("\n=== Benchmark Summary ===")
+    print(f"Target Skill: {benchmark['metadata']['target_skill']}")
+    print(f"Models Compared: {', '.join(benchmark['metadata']['models_compared'])}")
+    print(f"Evals Run: {benchmark['metadata']['evals_run']}")
+    print(f"Note: {benchmark['metadata']['note']}")
+
+    print("\n=== Model Summary ===")
+    for model, summary in benchmark['model_summary'].items():
+        pr = summary['pass_rate']['mean']
+        time = summary['time_seconds']['mean']
+        tokens = summary['tokens']['mean']
+        print(f"  {model}: {pr*100:.0f}% pass rate, {time:.1f}s, {tokens:.0f} tokens")
+
+    print("\n=== Comparison ===")
+    comp = benchmark['comparison']
+    print(f"  Pass Rate Delta: {comp['pass_rate_delta']}")
+    print(f"  Time Delta: {comp['time_delta']}")
+    print(f"  Token Delta: {comp['token_delta']}")
+    print(f"  Cost Efficiency: {comp['cost_efficiency']}")
+
+    print("\n=== Recommendations (data-driven) ===")
+    for rec in benchmark['recommendations']:
+        print(f"  [{rec['scenario']}] Use {rec['recommended_model']}: {rec['reason']}")
+
+    print("\n=== Notes ===")
+    for note in benchmark['notes']:
+        print(f"  - {note}")
+
+    # Write benchmark.json
+    benchmark_path = test_dir / "benchmark.json"
+    with open(benchmark_path, "w") as f:
+        json.dump(benchmark, f, indent=2)
+    print(f"\nWritten: {benchmark_path}")
+
+    # Write benchmark.md
+    markdown = generate_markdown(benchmark)
+    md_path = test_dir / "benchmark.md"
+    with open(md_path, "w") as f:
+        f.write(markdown)
+    print(f"Written: {md_path}")
+
+    return benchmark
+
+
+def main():
+    print("Creating test workspace with anonymous run IDs...")
+    test_dir = create_test_workspace()
+
+    print("\nRunning aggregation with mapping.json...")
+    benchmark = run_aggregation(test_dir)
+
+    print("\n=== Test Complete ===")
+    print(f"Test workspace: {test_dir}")
+    print("\nKey points verified:")
+    print("  1. Run directories use anonymous IDs (run-001, run-002, etc.)")
+    print("  2. Grading files do not contain model information")
+    print("  3. Mapping.json maps run IDs to models")
+    print("  4. Aggregation combines grading + mapping for benchmark")
+    print("\nYou can now test the report viewer:")
+    print(f"  python -m eval-viewer.generate_report {test_dir} --serve")
+
+
+if __name__ == "__main__":
+    main()