cli-jaw 0.1.0

package/skills_ref/hugging-face-evaluation/scripts/run_vllm_eval_job.py

@@ -0,0 +1,331 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "huggingface-hub>=0.26.0",
+#     "python-dotenv>=1.2.1",
+# ]
+# ///
+
+"""
+Submit vLLM-based evaluation jobs using the `hf jobs uv run` CLI.
+
+This wrapper constructs the appropriate command to execute vLLM evaluation scripts
+(lighteval or inspect-ai) on Hugging Face Jobs with GPU hardware.
+
+Unlike run_eval_job.py (which uses inference providers/APIs), this script runs
+models directly on the job's GPU using vLLM or HuggingFace Transformers.
+
+Usage:
+    python run_vllm_eval_job.py \\
+        --model meta-llama/Llama-3.2-1B \\
+        --task mmlu \\
+        --framework lighteval \\
+        --hardware a10g-small
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import Optional
+
+from huggingface_hub import get_token
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# Script paths for different evaluation frameworks
+SCRIPT_DIR = Path(__file__).parent.resolve()
+LIGHTEVAL_SCRIPT = SCRIPT_DIR / "lighteval_vllm_uv.py"
+INSPECT_SCRIPT = SCRIPT_DIR / "inspect_vllm_uv.py"
+
+# Hardware flavor recommendations for different model sizes
+HARDWARE_RECOMMENDATIONS = {
+    "small": "t4-small",       # < 3B parameters
+    "medium": "a10g-small",    # 3B - 13B parameters
+    "large": "a10g-large",     # 13B - 34B parameters
+    "xlarge": "a100-large",    # 34B+ parameters
+}
+
+
+def estimate_hardware(model_id: str) -> str:
+    """
+    Estimate appropriate hardware based on model ID naming conventions.
+    
+    Returns a hardware flavor recommendation.
+    """
+    model_lower = model_id.lower()
+    
+    # Check for explicit size indicators in model name
+    if any(x in model_lower for x in ["70b", "72b", "65b"]):
+        return "a100-large"
+    elif any(x in model_lower for x in ["34b", "33b", "32b", "30b"]):
+        return "a10g-large"
+    elif any(x in model_lower for x in ["13b", "14b", "7b", "8b"]):
+        return "a10g-small"
+    elif any(x in model_lower for x in ["3b", "2b", "1b", "0.5b", "small", "mini"]):
+        return "t4-small"
+    
+    # Default to medium hardware
+    return "a10g-small"
+
+
+def create_lighteval_job(
+    model_id: str,
+    tasks: str,
+    hardware: str,
+    hf_token: Optional[str] = None,
+    max_samples: Optional[int] = None,
+    backend: str = "vllm",
+    batch_size: int = 1,
+    tensor_parallel_size: int = 1,
+    trust_remote_code: bool = False,
+    use_chat_template: bool = False,
+) -> None:
+    """
+    Submit a lighteval evaluation job on HuggingFace Jobs.
+    """
+    token = hf_token or os.getenv("HF_TOKEN") or get_token()
+    if not token:
+        raise ValueError("HF_TOKEN is required. Set it in environment or pass as argument.")
+
+    if not LIGHTEVAL_SCRIPT.exists():
+        raise FileNotFoundError(f"Script not found at {LIGHTEVAL_SCRIPT}")
+
+    print(f"Preparing lighteval job for {model_id}")
+    print(f"  Tasks: {tasks}")
+    print(f"  Backend: {backend}")
+    print(f"  Hardware: {hardware}")
+
+    cmd = [
+        "hf", "jobs", "uv", "run",
+        str(LIGHTEVAL_SCRIPT),
+        "--flavor", hardware,
+        "--secrets", f"HF_TOKEN={token}",
+        "--",
+        "--model", model_id,
+        "--tasks", tasks,
+        "--backend", backend,
+        "--batch-size", str(batch_size),
+        "--tensor-parallel-size", str(tensor_parallel_size),
+    ]
+
+    if max_samples:
+        cmd.extend(["--max-samples", str(max_samples)])
+
+    if trust_remote_code:
+        cmd.append("--trust-remote-code")
+
+    if use_chat_template:
+        cmd.append("--use-chat-template")
+
+    print(f"\nExecuting: {' '.join(cmd)}")
+
+    try:
+        subprocess.run(cmd, check=True)
+    except subprocess.CalledProcessError as exc:
+        print("hf jobs command failed", file=sys.stderr)
+        raise
+
+
+def create_inspect_job(
+    model_id: str,
+    task: str,
+    hardware: str,
+    hf_token: Optional[str] = None,
+    limit: Optional[int] = None,
+    backend: str = "vllm",
+    tensor_parallel_size: int = 1,
+    trust_remote_code: bool = False,
+) -> None:
+    """
+    Submit an inspect-ai evaluation job on HuggingFace Jobs.
+    """
+    token = hf_token or os.getenv("HF_TOKEN") or get_token()
+    if not token:
+        raise ValueError("HF_TOKEN is required. Set it in environment or pass as argument.")
+
+    if not INSPECT_SCRIPT.exists():
+        raise FileNotFoundError(f"Script not found at {INSPECT_SCRIPT}")
+
+    print(f"Preparing inspect-ai job for {model_id}")
+    print(f"  Task: {task}")
+    print(f"  Backend: {backend}")
+    print(f"  Hardware: {hardware}")
+
+    cmd = [
+        "hf", "jobs", "uv", "run",
+        str(INSPECT_SCRIPT),
+        "--flavor", hardware,
+        "--secrets", f"HF_TOKEN={token}",
+        "--",
+        "--model", model_id,
+        "--task", task,
+        "--backend", backend,
+        "--tensor-parallel-size", str(tensor_parallel_size),
+    ]
+
+    if limit:
+        cmd.extend(["--limit", str(limit)])
+
+    if trust_remote_code:
+        cmd.append("--trust-remote-code")
+
+    print(f"\nExecuting: {' '.join(cmd)}")
+
+    try:
+        subprocess.run(cmd, check=True)
+    except subprocess.CalledProcessError as exc:
+        print("hf jobs command failed", file=sys.stderr)
+        raise
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Submit vLLM-based evaluation jobs to HuggingFace Jobs",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Run lighteval with vLLM on A10G GPU
+  python run_vllm_eval_job.py \\
+      --model meta-llama/Llama-3.2-1B \\
+      --task "leaderboard|mmlu|5" \\
+      --framework lighteval \\
+      --hardware a10g-small
+
+  # Run inspect-ai on larger model with multi-GPU
+  python run_vllm_eval_job.py \\
+      --model meta-llama/Llama-3.2-70B \\
+      --task mmlu \\
+      --framework inspect \\
+      --hardware a100-large \\
+      --tensor-parallel-size 4
+
+  # Auto-detect hardware based on model size
+  python run_vllm_eval_job.py \\
+      --model meta-llama/Llama-3.2-1B \\
+      --task mmlu \\
+      --framework inspect
+
+  # Run with HF Transformers backend (instead of vLLM)
+  python run_vllm_eval_job.py \\
+      --model microsoft/phi-2 \\
+      --task mmlu \\
+      --framework inspect \\
+      --backend hf
+
+Hardware flavors:
+  - t4-small: T4 GPU, good for models < 3B
+  - a10g-small: A10G GPU, good for models 3B-13B
+  - a10g-large: A10G GPU, good for models 13B-34B
+  - a100-large: A100 GPU, good for models 34B+
+
+Frameworks:
+  - lighteval: HuggingFace's lighteval library
+  - inspect: UK AI Safety's inspect-ai library
+
+Task formats:
+  - lighteval: "suite|task|num_fewshot" (e.g., "leaderboard|mmlu|5")
+  - inspect: task name (e.g., "mmlu", "gsm8k")
+        """,
+    )
+
+    parser.add_argument(
+        "--model",
+        required=True,
+        help="HuggingFace model ID (e.g., meta-llama/Llama-3.2-1B)",
+    )
+    parser.add_argument(
+        "--task",
+        required=True,
+        help="Evaluation task (format depends on framework)",
+    )
+    parser.add_argument(
+        "--framework",
+        choices=["lighteval", "inspect"],
+        default="lighteval",
+        help="Evaluation framework to use (default: lighteval)",
+    )
+    parser.add_argument(
+        "--hardware",
+        default=None,
+        help="Hardware flavor (auto-detected if not specified)",
+    )
+    parser.add_argument(
+        "--backend",
+        choices=["vllm", "hf", "accelerate"],
+        default="vllm",
+        help="Model backend (default: vllm)",
+    )
+    parser.add_argument(
+        "--limit",
+        "--max-samples",
+        type=int,
+        default=None,
+        dest="limit",
+        help="Limit number of samples to evaluate",
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=1,
+        help="Batch size for evaluation (lighteval only)",
+    )
+    parser.add_argument(
+        "--tensor-parallel-size",
+        type=int,
+        default=1,
+        help="Number of GPUs for tensor parallelism",
+    )
+    parser.add_argument(
+        "--trust-remote-code",
+        action="store_true",
+        help="Allow executing remote code from model repository",
+    )
+    parser.add_argument(
+        "--use-chat-template",
+        action="store_true",
+        help="Apply chat template (lighteval only)",
+    )
+
+    args = parser.parse_args()
+
+    # Auto-detect hardware if not specified
+    hardware = args.hardware or estimate_hardware(args.model)
+    print(f"Using hardware: {hardware}")
+
+    # Map backend names between frameworks
+    backend = args.backend
+    if args.framework == "lighteval" and backend == "hf":
+        backend = "accelerate"  # lighteval uses "accelerate" for HF backend
+
+    if args.framework == "lighteval":
+        create_lighteval_job(
+            model_id=args.model,
+            tasks=args.task,
+            hardware=hardware,
+            max_samples=args.limit,
+            backend=backend,
+            batch_size=args.batch_size,
+            tensor_parallel_size=args.tensor_parallel_size,
+            trust_remote_code=args.trust_remote_code,
+            use_chat_template=args.use_chat_template,
+        )
+    else:
+        create_inspect_job(
+            model_id=args.model,
+            task=args.task,
+            hardware=hardware,
+            limit=args.limit,
+            backend=backend if backend != "accelerate" else "hf",
+            tensor_parallel_size=args.tensor_parallel_size,
+            trust_remote_code=args.trust_remote_code,
+        )
+
+
+if __name__ == "__main__":
+    main()
+

package/skills_ref/hugging-face-evaluation/scripts/test_extraction.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "pyyaml",
+# ]
+# ///
+"""
+Test script for evaluation extraction functionality.
+
+This script demonstrates the table extraction capabilities without
+requiring HF tokens or making actual API calls.
+
+Note: This script imports from evaluation_manager.py (same directory).
+Run from the scripts/ directory: cd scripts && uv run test_extraction.py
+"""
+
+import yaml
+
+from evaluation_manager import (
+    extract_tables_from_markdown,
+    parse_markdown_table,
+    is_evaluation_table,
+    extract_metrics_from_table
+)
+
+# Sample README content with various table formats
+SAMPLE_README = """
+# My Awesome Model
+
+## Evaluation Results
+
+Here are the benchmark results:
+
+| Benchmark | Score |
+|-----------|-------|
+| MMLU      | 85.2  |
+| HumanEval | 72.5  |
+| GSM8K     | 91.3  |
+
+### Detailed Breakdown
+
+| Category      | MMLU  | GSM8K | HumanEval |
+|---------------|-------|-------|-----------|
+| Performance   | 85.2  | 91.3  | 72.5      |
+
+## Other Information
+
+This is not an evaluation table:
+
+| Feature | Value |
+|---------|-------|
+| Size    | 7B    |
+| Type    | Chat  |
+
+## More Results
+
+| Benchmark     | Accuracy | F1 Score |
+|---------------|----------|----------|
+| HellaSwag     | 88.9     | 0.87     |
+| TruthfulQA    | 68.7     | 0.65     |
+"""
+
+
+def test_table_extraction():
+    """Test markdown table extraction."""
+    print("=" * 60)
+    print("TEST 1: Table Extraction")
+    print("=" * 60)
+
+    tables = extract_tables_from_markdown(SAMPLE_README)
+    print(f"Found {len(tables)} tables in the sample README\n")
+
+    for i, table in enumerate(tables, 1):
+        print(f"Table {i}:")
+        print(table[:100] + "..." if len(table) > 100 else table)
+        print()
+
+    return tables
+
+
+def test_table_parsing(tables):
+    """Test table parsing."""
+    print("\n" + "=" * 60)
+    print("TEST 2: Table Parsing")
+    print("=" * 60)
+
+    parsed_tables = []
+    for i, table in enumerate(tables, 1):
+        print(f"\nParsing Table {i}:")
+        header, rows = parse_markdown_table(table)
+
+        print(f"  Header: {header}")
+        print(f"  Rows: {len(rows)}")
+        for j, row in enumerate(rows[:3], 1):  # Show first 3 rows
+            print(f"    Row {j}: {row}")
+        if len(rows) > 3:
+            print(f"    ... and {len(rows) - 3} more rows")
+
+        parsed_tables.append((header, rows))
+
+    return parsed_tables
+
+
+def test_evaluation_detection(parsed_tables):
+    """Test evaluation table detection."""
+    print("\n" + "=" * 60)
+    print("TEST 3: Evaluation Table Detection")
+    print("=" * 60)
+
+    eval_tables = []
+    for i, (header, rows) in enumerate(parsed_tables, 1):
+        is_eval = is_evaluation_table(header, rows)
+        status = "✓ IS" if is_eval else "✗ NOT"
+        print(f"\nTable {i}: {status} an evaluation table")
+        print(f"  Header: {header}")
+
+        if is_eval:
+            eval_tables.append((header, rows))
+
+    print(f"\nFound {len(eval_tables)} evaluation tables")
+    return eval_tables
+
+
+def test_metric_extraction(eval_tables):
+    """Test metric extraction."""
+    print("\n" + "=" * 60)
+    print("TEST 4: Metric Extraction")
+    print("=" * 60)
+
+    all_metrics = []
+    for i, (header, rows) in enumerate(eval_tables, 1):
+        print(f"\nExtracting metrics from table {i}:")
+        metrics = extract_metrics_from_table(header, rows, table_format="auto")
+
+        print(f"  Extracted {len(metrics)} metrics:")
+        for metric in metrics:
+            print(f"    - {metric['name']}: {metric['value']} (type: {metric['type']})")
+
+        all_metrics.extend(metrics)
+
+    return all_metrics
+
+
+def test_model_index_format(metrics):
+    """Test model-index format generation."""
+    print("\n" + "=" * 60)
+    print("TEST 5: Model-Index Format")
+    print("=" * 60)
+
+    model_index = {
+        "model-index": [
+            {
+                "name": "test-model",
+                "results": [
+                    {
+                        "task": {"type": "text-generation"},
+                        "dataset": {
+                            "name": "Benchmarks",
+                            "type": "benchmark"
+                        },
+                        "metrics": metrics,
+                        "source": {
+                            "name": "Model README",
+                            "url": "https://huggingface.co/test/model"
+                        }
+                    }
+                ]
+            }
+        ]
+    }
+
+    print("\nGenerated model-index structure:")
+    print(yaml.dump(model_index, sort_keys=False, default_flow_style=False))
+
+
+def main():
+    """Run all tests."""
+    print("\n" + "=" * 60)
+    print("EVALUATION EXTRACTION TEST SUITE")
+    print("=" * 60)
+    print("\nThis test demonstrates the table extraction capabilities")
+    print("without requiring API access or tokens.\n")
+
+    # Run tests
+    tables = test_table_extraction()
+    parsed_tables = test_table_parsing(tables)
+    eval_tables = test_evaluation_detection(parsed_tables)
+    metrics = test_metric_extraction(eval_tables)
+    test_model_index_format(metrics)
+
+    # Summary
+    print("\n" + "=" * 60)
+    print("TEST SUMMARY")
+    print("=" * 60)
+    print(f"✓ Found {len(tables)} total tables")
+    print(f"✓ Identified {len(eval_tables)} evaluation tables")
+    print(f"✓ Extracted {len(metrics)} metrics")
+    print("✓ Generated model-index format successfully")
+    print("\n" + "=" * 60)
+    print("All tests completed! The extraction logic is working correctly.")
+    print("=" * 60 + "\n")
+
+
+if __name__ == "__main__":
+    main()