cli-jaw 0.1.6 → 0.1.8

package/skills_ref/hugging-face-evaluation/scripts/run_vllm_eval_job.py

@@ -1,331 +0,0 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "huggingface-hub>=0.26.0",
-#     "python-dotenv>=1.2.1",
-# ]
-# ///
-
-"""
-Submit vLLM-based evaluation jobs using the `hf jobs uv run` CLI.
-
-This wrapper constructs the appropriate command to execute vLLM evaluation scripts
-(lighteval or inspect-ai) on Hugging Face Jobs with GPU hardware.
-
-Unlike run_eval_job.py (which uses inference providers/APIs), this script runs
-models directly on the job's GPU using vLLM or HuggingFace Transformers.
-
-Usage:
-    python run_vllm_eval_job.py \\
-        --model meta-llama/Llama-3.2-1B \\
-        --task mmlu \\
-        --framework lighteval \\
-        --hardware a10g-small
-"""
-
-from __future__ import annotations
-
-import argparse
-import os
-import subprocess
-import sys
-from pathlib import Path
-from typing import Optional
-
-from huggingface_hub import get_token
-from dotenv import load_dotenv
-
-load_dotenv()
-
-# Script paths for different evaluation frameworks
-SCRIPT_DIR = Path(__file__).parent.resolve()
-LIGHTEVAL_SCRIPT = SCRIPT_DIR / "lighteval_vllm_uv.py"
-INSPECT_SCRIPT = SCRIPT_DIR / "inspect_vllm_uv.py"
-
-# Hardware flavor recommendations for different model sizes
-HARDWARE_RECOMMENDATIONS = {
-    "small": "t4-small",       # < 3B parameters
-    "medium": "a10g-small",    # 3B - 13B parameters
-    "large": "a10g-large",     # 13B - 34B parameters
-    "xlarge": "a100-large",    # 34B+ parameters
-}
-
-
-def estimate_hardware(model_id: str) -> str:
-    """
-    Estimate appropriate hardware based on model ID naming conventions.
-    
-    Returns a hardware flavor recommendation.
-    """
-    model_lower = model_id.lower()
-    
-    # Check for explicit size indicators in model name
-    if any(x in model_lower for x in ["70b", "72b", "65b"]):
-        return "a100-large"
-    elif any(x in model_lower for x in ["34b", "33b", "32b", "30b"]):
-        return "a10g-large"
-    elif any(x in model_lower for x in ["13b", "14b", "7b", "8b"]):
-        return "a10g-small"
-    elif any(x in model_lower for x in ["3b", "2b", "1b", "0.5b", "small", "mini"]):
-        return "t4-small"
-    
-    # Default to medium hardware
-    return "a10g-small"
-
-
-def create_lighteval_job(
-    model_id: str,
-    tasks: str,
-    hardware: str,
-    hf_token: Optional[str] = None,
-    max_samples: Optional[int] = None,
-    backend: str = "vllm",
-    batch_size: int = 1,
-    tensor_parallel_size: int = 1,
-    trust_remote_code: bool = False,
-    use_chat_template: bool = False,
-) -> None:
-    """
-    Submit a lighteval evaluation job on HuggingFace Jobs.
-    """
-    token = hf_token or os.getenv("HF_TOKEN") or get_token()
-    if not token:
-        raise ValueError("HF_TOKEN is required. Set it in environment or pass as argument.")
-
-    if not LIGHTEVAL_SCRIPT.exists():
-        raise FileNotFoundError(f"Script not found at {LIGHTEVAL_SCRIPT}")
-
-    print(f"Preparing lighteval job for {model_id}")
-    print(f"  Tasks: {tasks}")
-    print(f"  Backend: {backend}")
-    print(f"  Hardware: {hardware}")
-
-    cmd = [
-        "hf", "jobs", "uv", "run",
-        str(LIGHTEVAL_SCRIPT),
-        "--flavor", hardware,
-        "--secrets", f"HF_TOKEN={token}",
-        "--",
-        "--model", model_id,
-        "--tasks", tasks,
-        "--backend", backend,
-        "--batch-size", str(batch_size),
-        "--tensor-parallel-size", str(tensor_parallel_size),
-    ]
-
-    if max_samples:
-        cmd.extend(["--max-samples", str(max_samples)])
-
-    if trust_remote_code:
-        cmd.append("--trust-remote-code")
-
-    if use_chat_template:
-        cmd.append("--use-chat-template")
-
-    print(f"\nExecuting: {' '.join(cmd)}")
-
-    try:
-        subprocess.run(cmd, check=True)
-    except subprocess.CalledProcessError as exc:
-        print("hf jobs command failed", file=sys.stderr)
-        raise
-
-
-def create_inspect_job(
-    model_id: str,
-    task: str,
-    hardware: str,
-    hf_token: Optional[str] = None,
-    limit: Optional[int] = None,
-    backend: str = "vllm",
-    tensor_parallel_size: int = 1,
-    trust_remote_code: bool = False,
-) -> None:
-    """
-    Submit an inspect-ai evaluation job on HuggingFace Jobs.
-    """
-    token = hf_token or os.getenv("HF_TOKEN") or get_token()
-    if not token:
-        raise ValueError("HF_TOKEN is required. Set it in environment or pass as argument.")
-
-    if not INSPECT_SCRIPT.exists():
-        raise FileNotFoundError(f"Script not found at {INSPECT_SCRIPT}")
-
-    print(f"Preparing inspect-ai job for {model_id}")
-    print(f"  Task: {task}")
-    print(f"  Backend: {backend}")
-    print(f"  Hardware: {hardware}")
-
-    cmd = [
-        "hf", "jobs", "uv", "run",
-        str(INSPECT_SCRIPT),
-        "--flavor", hardware,
-        "--secrets", f"HF_TOKEN={token}",
-        "--",
-        "--model", model_id,
-        "--task", task,
-        "--backend", backend,
-        "--tensor-parallel-size", str(tensor_parallel_size),
-    ]
-
-    if limit:
-        cmd.extend(["--limit", str(limit)])
-
-    if trust_remote_code:
-        cmd.append("--trust-remote-code")
-
-    print(f"\nExecuting: {' '.join(cmd)}")
-
-    try:
-        subprocess.run(cmd, check=True)
-    except subprocess.CalledProcessError as exc:
-        print("hf jobs command failed", file=sys.stderr)
-        raise
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(
-        description="Submit vLLM-based evaluation jobs to HuggingFace Jobs",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-Examples:
-  # Run lighteval with vLLM on A10G GPU
-  python run_vllm_eval_job.py \\
-      --model meta-llama/Llama-3.2-1B \\
-      --task "leaderboard|mmlu|5" \\
-      --framework lighteval \\
-      --hardware a10g-small
-
-  # Run inspect-ai on larger model with multi-GPU
-  python run_vllm_eval_job.py \\
-      --model meta-llama/Llama-3.2-70B \\
-      --task mmlu \\
-      --framework inspect \\
-      --hardware a100-large \\
-      --tensor-parallel-size 4
-
-  # Auto-detect hardware based on model size
-  python run_vllm_eval_job.py \\
-      --model meta-llama/Llama-3.2-1B \\
-      --task mmlu \\
-      --framework inspect
-
-  # Run with HF Transformers backend (instead of vLLM)
-  python run_vllm_eval_job.py \\
-      --model microsoft/phi-2 \\
-      --task mmlu \\
-      --framework inspect \\
-      --backend hf
-
-Hardware flavors:
-  - t4-small: T4 GPU, good for models < 3B
-  - a10g-small: A10G GPU, good for models 3B-13B
-  - a10g-large: A10G GPU, good for models 13B-34B
-  - a100-large: A100 GPU, good for models 34B+
-
-Frameworks:
-  - lighteval: HuggingFace's lighteval library
-  - inspect: UK AI Safety's inspect-ai library
-
-Task formats:
-  - lighteval: "suite|task|num_fewshot" (e.g., "leaderboard|mmlu|5")
-  - inspect: task name (e.g., "mmlu", "gsm8k")
-        """,
-    )
-
-    parser.add_argument(
-        "--model",
-        required=True,
-        help="HuggingFace model ID (e.g., meta-llama/Llama-3.2-1B)",
-    )
-    parser.add_argument(
-        "--task",
-        required=True,
-        help="Evaluation task (format depends on framework)",
-    )
-    parser.add_argument(
-        "--framework",
-        choices=["lighteval", "inspect"],
-        default="lighteval",
-        help="Evaluation framework to use (default: lighteval)",
-    )
-    parser.add_argument(
-        "--hardware",
-        default=None,
-        help="Hardware flavor (auto-detected if not specified)",
-    )
-    parser.add_argument(
-        "--backend",
-        choices=["vllm", "hf", "accelerate"],
-        default="vllm",
-        help="Model backend (default: vllm)",
-    )
-    parser.add_argument(
-        "--limit",
-        "--max-samples",
-        type=int,
-        default=None,
-        dest="limit",
-        help="Limit number of samples to evaluate",
-    )
-    parser.add_argument(
-        "--batch-size",
-        type=int,
-        default=1,
-        help="Batch size for evaluation (lighteval only)",
-    )
-    parser.add_argument(
-        "--tensor-parallel-size",
-        type=int,
-        default=1,
-        help="Number of GPUs for tensor parallelism",
-    )
-    parser.add_argument(
-        "--trust-remote-code",
-        action="store_true",
-        help="Allow executing remote code from model repository",
-    )
-    parser.add_argument(
-        "--use-chat-template",
-        action="store_true",
-        help="Apply chat template (lighteval only)",
-    )
-
-    args = parser.parse_args()
-
-    # Auto-detect hardware if not specified
-    hardware = args.hardware or estimate_hardware(args.model)
-    print(f"Using hardware: {hardware}")
-
-    # Map backend names between frameworks
-    backend = args.backend
-    if args.framework == "lighteval" and backend == "hf":
-        backend = "accelerate"  # lighteval uses "accelerate" for HF backend
-
-    if args.framework == "lighteval":
-        create_lighteval_job(
-            model_id=args.model,
-            tasks=args.task,
-            hardware=hardware,
-            max_samples=args.limit,
-            backend=backend,
-            batch_size=args.batch_size,
-            tensor_parallel_size=args.tensor_parallel_size,
-            trust_remote_code=args.trust_remote_code,
-            use_chat_template=args.use_chat_template,
-        )
-    else:
-        create_inspect_job(
-            model_id=args.model,
-            task=args.task,
-            hardware=hardware,
-            limit=args.limit,
-            backend=backend if backend != "accelerate" else "hf",
-            tensor_parallel_size=args.tensor_parallel_size,
-            trust_remote_code=args.trust_remote_code,
-        )
-
-
-if __name__ == "__main__":
-    main()
-

package/skills_ref/hugging-face-evaluation/scripts/test_extraction.py

@@ -1,206 +0,0 @@
-#!/usr/bin/env python3
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "pyyaml",
-# ]
-# ///
-"""
-Test script for evaluation extraction functionality.
-
-This script demonstrates the table extraction capabilities without
-requiring HF tokens or making actual API calls.
-
-Note: This script imports from evaluation_manager.py (same directory).
-Run from the scripts/ directory: cd scripts && uv run test_extraction.py
-"""
-
-import yaml
-
-from evaluation_manager import (
-    extract_tables_from_markdown,
-    parse_markdown_table,
-    is_evaluation_table,
-    extract_metrics_from_table
-)
-
-# Sample README content with various table formats
-SAMPLE_README = """
-# My Awesome Model
-
-## Evaluation Results
-
-Here are the benchmark results:
-
-| Benchmark | Score |
-|-----------|-------|
-| MMLU      | 85.2  |
-| HumanEval | 72.5  |
-| GSM8K     | 91.3  |
-
-### Detailed Breakdown
-
-| Category      | MMLU  | GSM8K | HumanEval |
-|---------------|-------|-------|-----------|
-| Performance   | 85.2  | 91.3  | 72.5      |
-
-## Other Information
-
-This is not an evaluation table:
-
-| Feature | Value |
-|---------|-------|
-| Size    | 7B    |
-| Type    | Chat  |
-
-## More Results
-
-| Benchmark     | Accuracy | F1 Score |
-|---------------|----------|----------|
-| HellaSwag     | 88.9     | 0.87     |
-| TruthfulQA    | 68.7     | 0.65     |
-"""
-
-
-def test_table_extraction():
-    """Test markdown table extraction."""
-    print("=" * 60)
-    print("TEST 1: Table Extraction")
-    print("=" * 60)
-
-    tables = extract_tables_from_markdown(SAMPLE_README)
-    print(f"Found {len(tables)} tables in the sample README\n")
-
-    for i, table in enumerate(tables, 1):
-        print(f"Table {i}:")
-        print(table[:100] + "..." if len(table) > 100 else table)
-        print()
-
-    return tables
-
-
-def test_table_parsing(tables):
-    """Test table parsing."""
-    print("\n" + "=" * 60)
-    print("TEST 2: Table Parsing")
-    print("=" * 60)
-
-    parsed_tables = []
-    for i, table in enumerate(tables, 1):
-        print(f"\nParsing Table {i}:")
-        header, rows = parse_markdown_table(table)
-
-        print(f"  Header: {header}")
-        print(f"  Rows: {len(rows)}")
-        for j, row in enumerate(rows[:3], 1):  # Show first 3 rows
-            print(f"    Row {j}: {row}")
-        if len(rows) > 3:
-            print(f"    ... and {len(rows) - 3} more rows")
-
-        parsed_tables.append((header, rows))
-
-    return parsed_tables
-
-
-def test_evaluation_detection(parsed_tables):
-    """Test evaluation table detection."""
-    print("\n" + "=" * 60)
-    print("TEST 3: Evaluation Table Detection")
-    print("=" * 60)
-
-    eval_tables = []
-    for i, (header, rows) in enumerate(parsed_tables, 1):
-        is_eval = is_evaluation_table(header, rows)
-        status = "✓ IS" if is_eval else "✗ NOT"
-        print(f"\nTable {i}: {status} an evaluation table")
-        print(f"  Header: {header}")
-
-        if is_eval:
-            eval_tables.append((header, rows))
-
-    print(f"\nFound {len(eval_tables)} evaluation tables")
-    return eval_tables
-
-
-def test_metric_extraction(eval_tables):
-    """Test metric extraction."""
-    print("\n" + "=" * 60)
-    print("TEST 4: Metric Extraction")
-    print("=" * 60)
-
-    all_metrics = []
-    for i, (header, rows) in enumerate(eval_tables, 1):
-        print(f"\nExtracting metrics from table {i}:")
-        metrics = extract_metrics_from_table(header, rows, table_format="auto")
-
-        print(f"  Extracted {len(metrics)} metrics:")
-        for metric in metrics:
-            print(f"    - {metric['name']}: {metric['value']} (type: {metric['type']})")
-
-        all_metrics.extend(metrics)
-
-    return all_metrics
-
-
-def test_model_index_format(metrics):
-    """Test model-index format generation."""
-    print("\n" + "=" * 60)
-    print("TEST 5: Model-Index Format")
-    print("=" * 60)
-
-    model_index = {
-        "model-index": [
-            {
-                "name": "test-model",
-                "results": [
-                    {
-                        "task": {"type": "text-generation"},
-                        "dataset": {
-                            "name": "Benchmarks",
-                            "type": "benchmark"
-                        },
-                        "metrics": metrics,
-                        "source": {
-                            "name": "Model README",
-                            "url": "https://huggingface.co/test/model"
-                        }
-                    }
-                ]
-            }
-        ]
-    }
-
-    print("\nGenerated model-index structure:")
-    print(yaml.dump(model_index, sort_keys=False, default_flow_style=False))
-
-
-def main():
-    """Run all tests."""
-    print("\n" + "=" * 60)
-    print("EVALUATION EXTRACTION TEST SUITE")
-    print("=" * 60)
-    print("\nThis test demonstrates the table extraction capabilities")
-    print("without requiring API access or tokens.\n")
-
-    # Run tests
-    tables = test_table_extraction()
-    parsed_tables = test_table_parsing(tables)
-    eval_tables = test_evaluation_detection(parsed_tables)
-    metrics = test_metric_extraction(eval_tables)
-    test_model_index_format(metrics)
-
-    # Summary
-    print("\n" + "=" * 60)
-    print("TEST SUMMARY")
-    print("=" * 60)
-    print(f"✓ Found {len(tables)} total tables")
-    print(f"✓ Identified {len(eval_tables)} evaluation tables")
-    print(f"✓ Extracted {len(metrics)} metrics")
-    print("✓ Generated model-index format successfully")
-    print("\n" + "=" * 60)
-    print("All tests completed! The extraction logic is working correctly.")
-    print("=" * 60 + "\n")
-
-
-if __name__ == "__main__":
-    main()