opencode-skills-collection 4.0.68 → 4.0.70

package/bundled-skills/huggingface-community-evals/scripts/lighteval_vllm_uv.py

@@ -0,0 +1,297 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "lighteval[accelerate,vllm]>=0.6.0",
+#     "torch>=2.0.0",
+#     "transformers>=4.40.0",
+#     "accelerate>=0.30.0",
+#     "vllm>=0.4.0",
+# ]
+# ///
+
+"""
+Entry point script for running lighteval evaluations with local GPU backends.
+
+This script runs evaluations using vLLM or accelerate on custom HuggingFace models.
+It is separate from inference provider scripts and evaluates models directly on local hardware.
+
+Usage (standalone):
+    uv run scripts/lighteval_vllm_uv.py --model "meta-llama/Llama-3.2-1B" --tasks "leaderboard|mmlu|5"
+
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+import sys
+from typing import Optional
+
+
+def setup_environment() -> None:
+    """Configure environment variables for HuggingFace authentication."""
+    hf_token = os.getenv("HF_TOKEN")
+    if hf_token:
+        os.environ.setdefault("HUGGING_FACE_HUB_TOKEN", hf_token)
+        os.environ.setdefault("HF_HUB_TOKEN", hf_token)
+
+
+def run_lighteval_vllm(
+    model_id: str,
+    tasks: str,
+    output_dir: Optional[str] = None,
+    max_samples: Optional[int] = None,
+    batch_size: int = 1,
+    tensor_parallel_size: int = 1,
+    gpu_memory_utilization: float = 0.8,
+    dtype: str = "auto",
+    trust_remote_code: bool = False,
+    use_chat_template: bool = False,
+    system_prompt: Optional[str] = None,
+) -> None:
+    """
+    Run lighteval with vLLM backend for efficient GPU inference.
+
+    Args:
+        model_id: HuggingFace model ID (e.g., "meta-llama/Llama-3.2-1B")
+        tasks: Task specification (e.g., "leaderboard|mmlu|5" or "lighteval|hellaswag|0")
+        output_dir: Directory for evaluation results
+        max_samples: Limit number of samples per task
+        batch_size: Batch size for evaluation
+        tensor_parallel_size: Number of GPUs for tensor parallelism
+        gpu_memory_utilization: GPU memory fraction to use (0.0-1.0)
+        dtype: Data type for model weights (auto, float16, bfloat16)
+        trust_remote_code: Allow executing remote code from model repo
+        use_chat_template: Apply chat template for conversational models
+        system_prompt: System prompt for chat models
+    """
+    setup_environment()
+
+    # Build lighteval vllm command
+    cmd = [
+        "lighteval",
+        "vllm",
+        model_id,
+        tasks,
+        "--batch-size", str(batch_size),
+        "--tensor-parallel-size", str(tensor_parallel_size),
+        "--gpu-memory-utilization", str(gpu_memory_utilization),
+        "--dtype", dtype,
+    ]
+
+    if output_dir:
+        cmd.extend(["--output-dir", output_dir])
+
+    if max_samples:
+        cmd.extend(["--max-samples", str(max_samples)])
+
+    if trust_remote_code:
+        cmd.append("--trust-remote-code")
+
+    if use_chat_template:
+        cmd.append("--use-chat-template")
+
+    if system_prompt:
+        cmd.extend(["--system-prompt", system_prompt])
+
+    print(f"Running: {' '.join(cmd)}")
+
+    try:
+        subprocess.run(cmd, check=True)
+        print("Evaluation complete.")
+    except subprocess.CalledProcessError as exc:
+        print(f"Evaluation failed with exit code {exc.returncode}", file=sys.stderr)
+        sys.exit(exc.returncode)
+
+
+def run_lighteval_accelerate(
+    model_id: str,
+    tasks: str,
+    output_dir: Optional[str] = None,
+    max_samples: Optional[int] = None,
+    batch_size: int = 1,
+    dtype: str = "bfloat16",
+    trust_remote_code: bool = False,
+    use_chat_template: bool = False,
+    system_prompt: Optional[str] = None,
+) -> None:
+    """
+    Run lighteval with accelerate backend for multi-GPU distributed inference.
+
+    Use this backend when vLLM is not available or for models not supported by vLLM.
+
+    Args:
+        model_id: HuggingFace model ID
+        tasks: Task specification
+        output_dir: Directory for evaluation results
+        max_samples: Limit number of samples per task
+        batch_size: Batch size for evaluation
+        dtype: Data type for model weights
+        trust_remote_code: Allow executing remote code
+        use_chat_template: Apply chat template
+        system_prompt: System prompt for chat models
+    """
+    setup_environment()
+
+    # Build lighteval accelerate command
+    cmd = [
+        "lighteval",
+        "accelerate",
+        model_id,
+        tasks,
+        "--batch-size", str(batch_size),
+        "--dtype", dtype,
+    ]
+
+    if output_dir:
+        cmd.extend(["--output-dir", output_dir])
+
+    if max_samples:
+        cmd.extend(["--max-samples", str(max_samples)])
+
+    if trust_remote_code:
+        cmd.append("--trust-remote-code")
+
+    if use_chat_template:
+        cmd.append("--use-chat-template")
+
+    if system_prompt:
+        cmd.extend(["--system-prompt", system_prompt])
+
+    print(f"Running: {' '.join(cmd)}")
+
+    try:
+        subprocess.run(cmd, check=True)
+        print("Evaluation complete.")
+    except subprocess.CalledProcessError as exc:
+        print(f"Evaluation failed with exit code {exc.returncode}", file=sys.stderr)
+        sys.exit(exc.returncode)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Run lighteval evaluations with vLLM or accelerate backend on custom HuggingFace models",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Run MMLU evaluation with vLLM
+  uv run scripts/lighteval_vllm_uv.py --model meta-llama/Llama-3.2-1B --tasks "leaderboard|mmlu|5"
+
+  # Run with accelerate backend instead of vLLM
+  uv run scripts/lighteval_vllm_uv.py --model meta-llama/Llama-3.2-1B --tasks "leaderboard|mmlu|5" --backend accelerate
+
+  # Run with chat template for instruction-tuned models
+  uv run scripts/lighteval_vllm_uv.py --model meta-llama/Llama-3.2-1B-Instruct --tasks "leaderboard|mmlu|5" --use-chat-template
+
+  # Run with limited samples for testing
+  uv run scripts/lighteval_vllm_uv.py --model meta-llama/Llama-3.2-1B --tasks "leaderboard|mmlu|5" --max-samples 10
+
+Task format:
+  Tasks use the format: "suite|task|num_fewshot"
+  - leaderboard|mmlu|5 (MMLU with 5-shot)
+  - lighteval|hellaswag|0 (HellaSwag zero-shot)
+  - leaderboard|gsm8k|5 (GSM8K with 5-shot)
+  - Multiple tasks: "leaderboard|mmlu|5,leaderboard|gsm8k|5"
+        """,
+    )
+
+    parser.add_argument(
+        "--model",
+        required=True,
+        help="HuggingFace model ID (e.g., meta-llama/Llama-3.2-1B)",
+    )
+    parser.add_argument(
+        "--tasks",
+        required=True,
+        help="Task specification (e.g., 'leaderboard|mmlu|5')",
+    )
+    parser.add_argument(
+        "--backend",
+        choices=["vllm", "accelerate"],
+        default="vllm",
+        help="Inference backend to use (default: vllm)",
+    )
+    parser.add_argument(
+        "--output-dir",
+        default=None,
+        help="Directory for evaluation results",
+    )
+    parser.add_argument(
+        "--max-samples",
+        type=int,
+        default=None,
+        help="Limit number of samples per task (useful for testing)",
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=1,
+        help="Batch size for evaluation (default: 1)",
+    )
+    parser.add_argument(
+        "--tensor-parallel-size",
+        type=int,
+        default=1,
+        help="Number of GPUs for tensor parallelism (vLLM only, default: 1)",
+    )
+    parser.add_argument(
+        "--gpu-memory-utilization",
+        type=float,
+        default=0.8,
+        help="GPU memory fraction to use (vLLM only, default: 0.8)",
+    )
+    parser.add_argument(
+        "--dtype",
+        default="auto",
+        choices=["auto", "float16", "bfloat16", "float32"],
+        help="Data type for model weights (default: auto)",
+    )
+    parser.add_argument(
+        "--trust-remote-code",
+        action="store_true",
+        help="Allow executing remote code from model repository",
+    )
+    parser.add_argument(
+        "--use-chat-template",
+        action="store_true",
+        help="Apply chat template for instruction-tuned/chat models",
+    )
+    parser.add_argument(
+        "--system-prompt",
+        default=None,
+        help="System prompt for chat models",
+    )
+
+    args = parser.parse_args()
+
+    if args.backend == "vllm":
+        run_lighteval_vllm(
+            model_id=args.model,
+            tasks=args.tasks,
+            output_dir=args.output_dir,
+            max_samples=args.max_samples,
+            batch_size=args.batch_size,
+            tensor_parallel_size=args.tensor_parallel_size,
+            gpu_memory_utilization=args.gpu_memory_utilization,
+            dtype=args.dtype,
+            trust_remote_code=args.trust_remote_code,
+            use_chat_template=args.use_chat_template,
+            system_prompt=args.system_prompt,
+        )
+    else:
+        run_lighteval_accelerate(
+            model_id=args.model,
+            tasks=args.tasks,
+            output_dir=args.output_dir,
+            max_samples=args.max_samples,
+            batch_size=args.batch_size,
+            dtype=args.dtype if args.dtype != "auto" else "bfloat16",
+            trust_remote_code=args.trust_remote_code,
+            use_chat_template=args.use_chat_template,
+            system_prompt=args.system_prompt,
+        )
+
+
+if __name__ == "__main__":
+    main()

package/bundled-skills/huggingface-datasets/SKILL.md

@@ -0,0 +1,130 @@
+---
+name: huggingface-datasets
+description: Use this skill for Hugging Face Dataset Viewer API workflows that fetch
+  subset/split metadata, paginate rows, search text, apply filters, download parquet
+  URLs, and read size or statistics.
+source_repo: huggingface/skills
+source_type: official
+source: huggingface
+date_added: '2026-09-21'
+risk: unknown
+---
+## When to Use
+- Use when this upstream workflow matches the user's stated goal.
+- Use when the task requires the procedures documented in this skill.
+
+# Hugging Face Dataset Viewer
+
+Use this skill to execute read-only Dataset Viewer API calls for dataset exploration and extraction.
+
+## Core workflow
+
+1. Optionally validate dataset availability with `/is-valid`.
+2. Resolve `config` + `split` with `/splits`.
+3. Preview with `/first-rows`.
+4. Paginate content with `/rows` using `offset` and `length` (max 100).
+5. Use `/search` for text matching and `/filter` for row predicates.
+6. Retrieve parquet links via `/parquet` and totals/metadata via `/size` and `/statistics`.
+
+## Defaults
+
+- Base URL: `https://datasets-server.huggingface.co`
+- Default API method: `GET`
+- Query params should be URL-encoded.
+- `offset` is 0-based.
+- `length` max is usually `100` for row-like endpoints.
+- Gated/private datasets require `Authorization: Bearer <HF_TOKEN>`.
+
+## Dataset Viewer
+
+- `Validate dataset`: `/is-valid?dataset=<namespace/repo>`
+- `List subsets and splits`: `/splits?dataset=<namespace/repo>`
+- `Preview first rows`: `/first-rows?dataset=<namespace/repo>&config=<config>&split=<split>`
+- `Paginate rows`: `/rows?dataset=<namespace/repo>&config=<config>&split=<split>&offset=<int>&length=<int>`
+- `Search text`: `/search?dataset=<namespace/repo>&config=<config>&split=<split>&query=<text>&offset=<int>&length=<int>`
+- `Filter with predicates`: `/filter?dataset=<namespace/repo>&config=<config>&split=<split>&where=<predicate>&orderby=<sort>&offset=<int>&length=<int>`
+- `List parquet shards`: `/parquet?dataset=<namespace/repo>`
+- `Get size totals`: `/size?dataset=<namespace/repo>`
+- `Get column statistics`: `/statistics?dataset=<namespace/repo>&config=<config>&split=<split>`
+- `Get Croissant metadata (if available)`: `/croissant?dataset=<namespace/repo>`
+
+Pagination pattern:
+
+```bash
+curl "https://datasets-server.huggingface.co/rows?dataset=stanfordnlp/imdb&config=plain_text&split=train&offset=0&length=100"
+curl "https://datasets-server.huggingface.co/rows?dataset=stanfordnlp/imdb&config=plain_text&split=train&offset=100&length=100"
+```
+
+When pagination is partial, use response fields such as `num_rows_total`, `num_rows_per_page`, and `partial` to drive continuation logic.
+
+Search/filter notes:
+
+- `/search` matches string columns (full-text style behavior is internal to the API).
+- `/filter` requires predicate syntax in `where` and optional sort in `orderby`.
+- Keep filtering and searches read-only and side-effect free.
+
+For CLI-based parquet URL discovery or SQL, use the `hf-cli` skill with `hf datasets parquet` and `hf datasets sql`.
+
+## Creating and Uploading Datasets
+
+Use one of these flows depending on dependency constraints.
+
+Zero local dependencies (Hub UI):
+
+- Create dataset repo in browser: `https://huggingface.co/new-dataset`
+- Upload parquet files in the repo "Files and versions" page.
+- Verify shards appear in Dataset Viewer:
+
+```bash
+curl -s "https://datasets-server.huggingface.co/parquet?dataset=<namespace>/<repo>"
+```
+
+Low dependency CLI flow (`npx @huggingface/hub` / `hfjs`):
+
+- Set auth token:
+
+```bash
+export HF_TOKEN=<your_hf_token>
+```
+
+- Upload parquet folder to a dataset repo (auto-creates repo if missing):
+
+```bash
+npx -y @huggingface/hub upload datasets/<namespace>/<repo> ./local/parquet-folder data
+```
+
+- Upload as private repo on creation:
+
+```bash
+npx -y @huggingface/hub upload datasets/<namespace>/<repo> ./local/parquet-folder data --private
+```
+
+After upload, call `/parquet` to discover `<config>/<split>/<shard>` values for querying with `@~parquet`.
+
+## Agent Traces
+
+The Hub supports raw agent session traces from Claude Code, Codex, and Pi Agent. Upload them to Hugging Face Datasets as original JSONL files and the Hub can auto-detect the trace format, tag the dataset as `Traces`, and enable the trace viewer for browsing sessions, turns, tool calls, and model responses. Common local session directories:
+
+- Claude Code: `~/.claude/projects`
+- Codex: `~/.codex/sessions`
+- Pi: `~/.pi/agent/sessions`
+
+Default to private dataset repos because traces can contain prompts, file paths, tool outputs, secrets, or PII. Preserve the raw `.jsonl` files and nest them by project/cwd instead of uploading every session at the dataset root.
+
+```bash
+hf repos create <namespace>/<repo> --type dataset --private --exist-ok
+hf upload <namespace>/<repo> ~/.codex/sessions codex/<project-or-cwd> --type dataset
+```
+
+
+## Examples
+
+```text
+User: Apply this skill to my current task.
+Assistant: Follow the workflow in this skill, cite limitations, and ask before risky steps.
+```
+
+## Limitations
+
+- Imported upstream skill; verify credentials, permissions, and safety boundaries before execution.
+- Does not replace environment-specific validation, testing, or maintainer review.