opencode-skills-collection 3.1.11 → 3.1.13

package/bundled-skills/train-sentence-transformers/scripts/train_sparse_encoder_example.py

@@ -0,0 +1,237 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "sentence-transformers[train]>=5.0",
+#     "datasets>=2.19.0",
+#     "accelerate>=0.26.0",
+#     "trackio",
+# ]
+# ///
+"""Production-ready sparse-encoder (SPLADE) training template.
+
+Demonstrates:
+- SpladeLoss wrapping SparseMultipleNegativesRankingLoss
+- FLOPS regularization (`query_regularizer_weight` / `document_regularizer_weight`)
+- SparseNanoBEIREvaluator for sparse retrieval metrics
+- load_best_model_at_end on the retrieval metric
+
+Base model must expose a masked-LM head; any `AutoModelForMaskedLM`-compatible
+checkpoint works (DistilBERT, BERT, MiniLM MLM variants, existing SPLADE models).
+
+Run locally:
+    pip install "sentence-transformers[train]>=5.0"
+    python train_sparse_encoder_example.py
+
+Multi-GPU:
+    accelerate launch train_sparse_encoder_example.py
+
+Hugging Face Jobs: paste this file's contents as the `script` in hf_jobs(...).
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+from contextlib import nullcontext
+
+import torch
+from datasets import load_dataset
+
+from sentence_transformers import (
+    SparseEncoder,
+    SparseEncoderModelCardData,
+    SparseEncoderTrainer,
+    SparseEncoderTrainingArguments,
+)
+from sentence_transformers.base.sampler import BatchSamplers
+from sentence_transformers.sparse_encoder.evaluation import SparseNanoBEIREvaluator
+from sentence_transformers.sparse_encoder.losses import (
+    SparseMultipleNegativesRankingLoss,
+    SpladeLoss,
+)
+
+
+def autocast_ctx():
+    """bf16/fp16 autocast for evaluator calls outside the trainer (which has its own autocast)."""
+    if not torch.cuda.is_available():
+        return nullcontext()
+    dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
+    return torch.autocast("cuda", dtype=dtype)
+
+
+def log_trackio_dashboard():
+    """Surface the Trackio dashboard URL so the user can watch training live."""
+    try:
+        from huggingface_hub import whoami
+
+        hf_user = whoami().get("name")
+        if hf_user:
+            logging.info(
+                f"Trackio dashboard (live training progress): https://huggingface.co/spaces/{hf_user}/trackio"
+            )
+    except Exception:
+        pass
+
+
+MODEL_NAME = "distilbert/distilbert-base-uncased"
+DATASET_NAME = "sentence-transformers/gooaq"
+TRAIN_SIZE = 100_000
+EVAL_SIZE = 1_000
+OUTPUT_DIR = "models/distilbert-splade-gooaq"
+RUN_NAME = "distilbert-splade-gooaq"
+
+QUERY_REGULARIZER_WEIGHT = 5e-5
+DOCUMENT_REGULARIZER_WEIGHT = 3e-5
+SMOKE_TEST = os.environ.get("SMOKE_TEST") == "1"
+
+
+def setup_logging():
+    """Configure logging + TF32. Tees to logs/{RUN_NAME}.log and silences HTTP spam."""
+    os.makedirs("logs", exist_ok=True)
+    logging.basicConfig(
+        format="%(asctime)s - %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+        level=logging.INFO,
+        handlers=[logging.StreamHandler(), logging.FileHandler(f"logs/{RUN_NAME}.log")],
+        force=True,
+    )
+    for noisy in ("httpx", "httpcore", "huggingface_hub", "urllib3", "filelock", "fsspec"):
+        logging.getLogger(noisy).setLevel(logging.WARNING)
+    if torch.cuda.is_available():
+        torch.set_float32_matmul_precision("high")  # TF32 on Ampere+, no quality loss
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--eval-only", type=str, default=None, help="Skip training; load this saved model and run only the evaluator."
+    )
+    cli, _ = parser.parse_known_args()
+
+    setup_logging()
+
+    if cli.eval_only:
+        logging.info(f"Eval-only mode: loading model from {cli.eval_only}")
+        model = SparseEncoder(cli.eval_only)
+        evaluator = SparseNanoBEIREvaluator(dataset_names=["msmarco", "nfcorpus", "nq"])
+        with autocast_ctx():
+            evaluator(model)
+        return
+
+    logging.info(f"Loading base model: {MODEL_NAME}")
+    # Prompts are optional for SPLADE: most BERT-style MLM bases don't need them.
+    # If you're starting from a CSR (`Transformer + Pooling + SparseAutoEncoder`)
+    # base like `tomaarsen/csr-mxbai-embed-large-v1-nq` that *was* trained with
+    # prompts, mirror them here to preserve quality:
+    #   prompts={"query": "Represent this sentence for similarity: ", "document": ""},
+    #   default_prompt_name="document",
+    model = SparseEncoder(
+        MODEL_NAME,
+        model_card_data=SparseEncoderModelCardData(
+            language="en",
+            license="apache-2.0",
+            model_name=f"SPLADE from {MODEL_NAME.split('/')[-1]} trained on GooAQ",
+        ),
+    )
+
+    logging.info(f"Loading dataset: {DATASET_NAME}")
+    train_size = 50 if SMOKE_TEST else TRAIN_SIZE
+    eval_size = 20 if SMOKE_TEST else EVAL_SIZE
+    if SMOKE_TEST:
+        logging.info("SMOKE_TEST=1: trimmed dataset; will run max_steps=1 and skip Hub push")
+    full = load_dataset(DATASET_NAME, split="train")
+    split = full.train_test_split(test_size=eval_size, seed=12)
+    train_dataset = split["train"].select(range(min(train_size, len(split["train"]))))
+    eval_dataset = split["test"]
+    logging.info(f"  train: {len(train_dataset):,} rows | eval: {len(eval_dataset):,} rows")
+    logging.info(f"  columns: {train_dataset.column_names}")
+
+    loss = SpladeLoss(
+        model=model,
+        loss=SparseMultipleNegativesRankingLoss(model=model),
+        query_regularizer_weight=QUERY_REGULARIZER_WEIGHT,
+        document_regularizer_weight=DOCUMENT_REGULARIZER_WEIGHT,
+    )
+
+    evaluator = SparseNanoBEIREvaluator(dataset_names=["msmarco", "nfcorpus", "nq"])
+    logging.info("Baseline evaluation:")
+    with autocast_ctx():
+        # Must run before deriving metric_key: evaluator(model) mutates primary_metric to add the name_ prefix.
+        baseline_result = evaluator(model)
+        baseline_eval = baseline_result[evaluator.primary_metric]
+    metric_key = f"eval_{evaluator.primary_metric}"
+
+    args = SparseEncoderTrainingArguments(
+        output_dir=OUTPUT_DIR,
+        num_train_epochs=1,
+        max_steps=1 if SMOKE_TEST else -1,
+        per_device_train_batch_size=32,
+        per_device_eval_batch_size=32,
+        learning_rate=2e-5,
+        weight_decay=0.01,
+        warmup_steps=0.1,
+        lr_scheduler_type="linear",
+        bf16=True,
+        batch_sampler=BatchSamplers.NO_DUPLICATES,
+        eval_strategy="steps",
+        eval_steps=0.1,
+        save_strategy="steps",
+        save_steps=0.1,
+        save_total_limit=2,
+        logging_steps=0.01,
+        logging_first_step=True,
+        load_best_model_at_end=True,
+        metric_for_best_model=metric_key,
+        greater_is_better=True,
+        report_to="none" if SMOKE_TEST else "trackio",
+        run_name=RUN_NAME,
+        seed=12,
+    )
+
+    trainer = SparseEncoderTrainer(
+        model=model,
+        args=args,
+        train_dataset=train_dataset,
+        eval_dataset=eval_dataset,
+        loss=loss,
+        evaluator=evaluator,
+    )
+    if not SMOKE_TEST:
+        log_trackio_dashboard()
+    trainer.train()
+
+    logging.info("Post-training evaluation:")
+    with autocast_ctx():
+        result = evaluator(model)
+        score = result[evaluator.primary_metric]
+    delta = score - baseline_eval
+    verdict = "WIN" if delta >= 0.005 else "MARGINAL" if delta >= 0 else "REGRESSION"
+    # Active-dim keys come back name-prefixed (e.g. "NanoBEIR_..._query_active_dims"); suffix-match for compat.
+    qad = next((v for k, v in result.items() if k.endswith("query_active_dims")), "n/a")
+    cad = next((v for k, v in result.items() if k.endswith("corpus_active_dims")), "n/a")
+    logging.info(
+        f"VERDICT: {verdict} | score={score:.4f} | baseline={baseline_eval:.4f} | delta={delta:+.4f} "
+        f"| query_active={qad} corpus_active={cad}"
+    )
+
+    final_dir = f"{OUTPUT_DIR}/final"
+    model.save_pretrained(final_dir)
+    logging.info(f"Saved final model to {final_dir}")
+
+    if SMOKE_TEST:
+        logging.info("SMOKE_TEST=1: skipping Hub push")
+        return
+
+    try:
+        commit_url = model.push_to_hub(RUN_NAME)
+        logging.info(f"Pushed model to {commit_url.rsplit('/commit/', 1)[0]}")
+    except Exception:
+        import traceback
+
+        logging.error(f"Hub push failed:\n{traceback.format_exc()}")
+
+
+if __name__ == "__main__":
+    main()

package/bundled-skills/transformers-js/SKILL.md

@@ -1,20 +1,18 @@
 ---
-source: "https://github.com/huggingface/skills/tree/main/skills/transformers-js"
 name: transformers-js
-description: Run Hugging Face models in JavaScript or TypeScript with Transformers.js in Node.js or the browser.
-license: Apache-2.0
+description: Use Transformers.js to run state-of-the-art machine learning models directly in JavaScript/TypeScript. Supports NLP (text classification, translation, summarization), computer vision (image classification, object detection), audio (speech recognition, audio classification), and...
 risk: unknown
-metadata:
-  author: huggingface
-  version: "3.8.1"
-  category: machine-learning
-  repository: https://github.com/huggingface/transformers.js
-compatibility: Requires Node.js 18+ or modern browser with ES modules support. WebGPU support requires compatible browser/environment. Internet access needed for downloading models from Hugging Face Hub (optional if using local models).
+source: https://github.com/huggingface/skills/tree/main/skills/transformers-js
+source_repo: huggingface/skills
+source_type: official
+date_added: 2026-07-01
+license: Apache-2.0
+license_source: https://github.com/huggingface/skills/blob/main/LICENSE
 ---
 
 # Transformers.js - Machine Learning for JavaScript
 
-Transformers.js enables running state-of-the-art machine learning models directly in JavaScript, both in browsers and Node.js environments, with no server required.
+Transformers.js enables running state-of-the-art machine learning models directly in JavaScript across browsers and server-side runtimes (Node.js, Bun, Deno), with no Python server required.
 
 ## When to Use This Skill
 
@@ -55,7 +53,7 @@ const result = await pipe('I love transformers!');
 // Output: [{ label: 'POSITIVE', score: 0.999817686 }]
 
 // IMPORTANT: Always dispose when done to free memory
-await classifier.dispose();
+await pipe.dispose();
 ```
 
 **⚠️ Memory Management:** All pipelines must be disposed with `pipe.dispose()` when finished to prevent memory leaks. See examples in [Code Examples](./references/EXAMPLES.md) for cleanup patterns across different environments.
@@ -88,7 +86,7 @@ Choose where to run the model:
 // Run on CPU (default for WASM)
 const pipe = await pipeline('sentiment-analysis', 'model-id');
 
-// Run on GPU (WebGPU - experimental)
+// Run on GPU (WebGPU)
 const pipe = await pipeline('sentiment-analysis', 'model-id', {
   device: 'webgpu',
 });
@@ -361,7 +359,7 @@ await generator.dispose();
 2. **Check ONNX Support**: Ensure the model has ONNX files (look for `onnx` folder in model repo)
 3. **Read Model Cards**: Model cards contain usage examples, limitations, and benchmarks
 4. **Test Locally**: Benchmark inference speed and memory usage in your environment
-5. **Community Models**: Look for models by `Xenova` (Transformers.js maintainer) or `onnx-community`
+5. **Filter by Library**: Use `library=transformers.js` to find compatible models: https://huggingface.co/models?library=transformers.js
 6. **Version Pin**: Use specific git commits in production for stability:
    ```javascript
    const pipe = await pipeline('task', 'model-id', { revision: 'abc123' });
@@ -376,10 +374,10 @@ The `env` object provides comprehensive control over Transformers.js execution,
 **Quick Overview:**
 
 ```javascript
-import { env } from '@huggingface/transformers';
+import { env, LogLevel } from '@huggingface/transformers';
 
 // View version
-console.log(env.version); // e.g., '3.8.1'
+console.log(env.version); // e.g., '4.x'
 
 // Common settings
 env.allowRemoteModels = true;  // Load from Hugging Face Hub
@@ -388,6 +386,18 @@ env.localModelPath = '/models/'; // Local model directory
 env.useFSCache = true;         // Cache models on disk (Node.js)
 env.useBrowserCache = true;    // Cache models in browser
 env.cacheDir = './.cache';     // Cache directory location
+// Optional: override logging level (default is LogLevel.WARNING)
+env.logLevel = LogLevel.INFO;
+
+// Optional: custom fetch for auth headers, retries, abort signals, etc.
+env.fetch = (url, options) =>
+  fetch(url, {
+    ...options,
+    headers: {
+      ...options?.headers,
+      Authorization: `Bearer ${HF_TOKEN}`,
+    },
+  });
 ```
 
 **Configuration Patterns:**
@@ -414,6 +424,43 @@ For complete documentation on all configuration options, caching strategies, cac
 
 **→ [Configuration Reference](./references/CONFIGURATION.md)**
 
+### ModelRegistry (v4)
+
+`ModelRegistry` gives you visibility and control over model assets before loading a pipeline. Use it to estimate download size, check cache status, inspect available dtypes, and clear cached artifacts for a specific task/model/options tuple.
+
+```javascript
+import { ModelRegistry } from '@huggingface/transformers';
+
+const task = 'feature-extraction';
+const modelId = 'onnx-community/all-MiniLM-L6-v2-ONNX';
+const modelOptions = { dtype: 'fp32' };
+
+// List required files for this pipeline
+const files = await ModelRegistry.get_pipeline_files(task, modelId, modelOptions);
+
+// Check if assets are already cached
+const cached = await ModelRegistry.is_pipeline_cached(task, modelId, modelOptions);
+
+// Inspect precision formats available for this model
+const dtypes = await ModelRegistry.get_available_dtypes(modelId);
+
+console.log({ files: files.length, cached, dtypes });
+```
+
+For production patterns and full API coverage, see **[ModelRegistry Reference](./references/MODEL_REGISTRY.md)**.
+
+### Standalone Tokenization (`@huggingface/tokenizers`)
+
+For tokenization-only workflows, use `@huggingface/tokenizers`. It is a separate lightweight package useful when you need fast tokenization/encoding without loading full model inference pipelines.
+
+```bash
+npm install @huggingface/tokenizers
+```
+
+```javascript
+import { Tokenizer } from '@huggingface/tokenizers';
+```
+
 ### Working with Tensors
 
 ```javascript
@@ -443,10 +490,10 @@ const results = await classifier([
 ]);
 ```
 
-## Browser-Specific Considerations
+## Runtime-Specific Considerations
 
 ### WebGPU Usage
-WebGPU provides GPU acceleration in browsers:
+WebGPU provides GPU acceleration in browsers and server-side runtimes (when supported):
 
 ```javascript
 const pipe = await pipeline('text-generation', 'onnx-community/gemma-3-270m-it-ONNX', {
@@ -455,10 +502,10 @@ const pipe = await pipeline('text-generation', 'onnx-community/gemma-3-270m-it-O
 });
 ```
 
-**Note**: WebGPU is experimental. Check browser compatibility and file issues if problems occur.
+**Note**: Use `webgpu` when available and fall back to WASM/CPU when not supported in the current runtime.
 
 ### WASM Performance
-Default browser execution uses WASM:
+WASM is the most compatible execution backend across runtimes:
 
 ```javascript
 // Optimized for browsers with quantization
@@ -478,13 +525,18 @@ import { pipeline } from '@huggingface/transformers';
 const fileProgress = {};
 
 function onProgress(info) {
-  console.log(`${info.status}: ${info.file}`);
-  
+  if (info.status === 'progress_total') {
+    console.log(`Total: ${info.progress.toFixed(1)}%`);
+    return;
+  }
+
+  console.log(`${info.status}: ${info.file ?? ''}`);
+
   if (info.status === 'progress') {
     fileProgress[info.file] = info.progress;
     console.log(`${info.file}: ${info.progress.toFixed(1)}%`);
   }
-  
+
   if (info.status === 'done') {
     console.log(`✓ ${info.file} complete`);
   }
@@ -500,10 +552,10 @@ const classifier = await pipeline('sentiment-analysis', null, {
 
 ```typescript
 interface ProgressInfo {
-  status: 'initiate' | 'download' | 'progress' | 'done' | 'ready';
+  status: 'initiate' | 'download' | 'progress' | 'progress_total' | 'done' | 'ready';
   name: string;      // Model id or path
-  file: string;      // File being processed
-  progress?: number; // Percentage (0-100, only for 'progress' status)
+  file?: string;     // File being processed (per-file events)
+  progress?: number; // Percentage (0-100, for 'progress' and 'progress_total')
   loaded?: number;   // Bytes downloaded (only for 'progress' status)
   total?: number;    // Total bytes (only for 'progress' status)
 }
@@ -581,6 +633,7 @@ For detailed patterns (React cleanup, servers, browser), see **[Code Examples](.
 ### This Skill
 - **[Pipeline Options](./references/PIPELINE_OPTIONS.md)** - Configure `pipeline()` with `progress_callback`, `device`, `dtype`, etc.
 - **[Configuration Reference](./references/CONFIGURATION.md)** - Global `env` configuration for caching and model loading
+- **[ModelRegistry Reference](./references/MODEL_REGISTRY.md)** - Inspect files, cache status, dtypes, and clear cache before loading pipelines
 - **[Caching Reference](./references/CACHE.md)** - Browser Cache API, Node.js filesystem cache, and custom cache implementations
 - **[Text Generation Guide](./references/TEXT_GENERATION.md)** - Streaming, chat format, and generation parameters
 - **[Model Architectures](./references/MODEL_ARCHITECTURES.md)** - Supported models and selection tips
@@ -591,7 +644,7 @@ For detailed patterns (React cleanup, servers, browser), see **[Code Examples](.
 - API reference: https://huggingface.co/docs/transformers.js/api/pipelines
 - Model hub: https://huggingface.co/models?library=transformers.js
 - GitHub: https://github.com/huggingface/transformers.js
-- Examples: https://github.com/huggingface/transformers.js/tree/main/examples
+- Examples: https://github.com/huggingface/transformers.js-examples
 
 ## Best Practices
 
@@ -639,6 +692,7 @@ For detailed patterns (React cleanup, servers, browser), see **[Code Examples](.
 This skill enables you to integrate state-of-the-art machine learning capabilities directly into JavaScript applications without requiring separate ML servers or Python environments.
 
 ## Limitations
-- Use this skill only when the task clearly matches the scope described above.
-- Do not treat the output as a substitute for environment-specific validation, testing, or expert review.
-- Stop and ask for clarification if required inputs, permissions, safety boundaries, or success criteria are missing.
+
+- Use this skill only when the task clearly matches its upstream product or API scope.
+- Verify commands, API behavior, pricing, quotas, credentials, and deployment effects against current official documentation before making changes.
+- Do not treat generated examples as a substitute for environment-specific tests, security review, or user approval for destructive or costly actions.

package/bundled-skills/transformers-js/references/CONFIGURATION.md

@@ -9,8 +9,9 @@ Complete guide to configuring Transformers.js behavior using the `env` object.
 3. [Local Model Configuration](#local-model-configuration)
 4. [Cache Configuration](#cache-configuration)
 5. [WASM Configuration](#wasm-configuration)
-6. [Common Configuration Patterns](#common-configuration-patterns)
-7. [Environment Best Practices](#environment-best-practices)
+6. [Network and Logging Controls](#network-and-logging-controls)
+7. [Common Configuration Patterns](#common-configuration-patterns)
+8. [Environment Best Practices](#environment-best-practices)
 
 ## Overview
 
@@ -20,7 +21,7 @@ The `env` object provides comprehensive control over Transformers.js execution,
 import { env } from '@huggingface/transformers';
 
 // View current version
-console.log(env.version); // e.g., '3.8.1'
+console.log(env.version); // e.g., '4.x'
 ```
 
 ### Available Properties
@@ -29,22 +30,22 @@ console.log(env.version); // e.g., '3.8.1'
 interface TransformersEnvironment {
   // Version info
   version: string;
-  
+
   // Backend configuration
   backends: {
     onnx: Partial<ONNXEnv>;
   };
-  
+
   // Remote model settings
   allowRemoteModels: boolean;
   remoteHost: string;
   remotePathTemplate: string;
-  
+
   // Local model settings
   allowLocalModels: boolean;
   localModelPath: string;
   useFS: boolean;
-  
+
   // Cache settings
   useBrowserCache: boolean;
   useFSCache: boolean;
@@ -53,6 +54,10 @@ interface TransformersEnvironment {
   customCache: CacheInterface | null;
   useWasmCache: boolean;
   cacheKey: string;
+
+  // Networking and logging (v4)
+  fetch: typeof globalThis.fetch;
+  logLevel: LogLevel;
 }
 ```
 
@@ -243,6 +248,49 @@ env.backends.onnx.wasm.wasmPaths = '/static/wasm/';
 - `ort-wasm-threaded.wasm` - Multi-threaded WASM binary
 - `ort-wasm-simd-threaded.wasm` - SIMD + multi-threaded WASM binary
 
+## Network and Logging Controls
+
+Transformers.js v4 adds environment controls for authenticated fetching and cleaner runtime logs.
+
+### Custom Fetch (`env.fetch`)
+
+Use `env.fetch` to inject auth headers, retries, custom routing, or abort handling.
+
+```javascript
+import { env } from '@huggingface/transformers';
+
+const HF_TOKEN = process.env.HF_TOKEN;
+
+env.fetch = (url, options) =>
+  fetch(url, {
+    ...options,
+    headers: {
+      ...options?.headers,
+      Authorization: `Bearer ${HF_TOKEN}`,
+    },
+  });
+```
+
+### Logging Level (`env.logLevel`)
+
+Use `env.logLevel` to override runtime verbosity. The default is `LogLevel.WARNING`.
+
+```javascript
+import { env, LogLevel } from '@huggingface/transformers';
+
+// Enable more detailed logs during development
+env.logLevel = LogLevel.INFO;
+```
+
+Common values:
+- `LogLevel.DEBUG`
+- `LogLevel.INFO`
+- `LogLevel.WARNING`
+- `LogLevel.ERROR`
+- `LogLevel.NONE`
+
+For ONNX Runtime session-level logging controls, see `session_options` in **[Pipeline Options](./PIPELINE_OPTIONS.md)**.
+
 ## Common Configuration Patterns
 
 ### Development Setup