opencode-skills-collection 3.1.12 → 3.1.14

package/bundled-skills/transformers-js/references/EXAMPLES.md

@@ -30,10 +30,10 @@ All examples use the same task and model for consistency:
   <div id="loading" style="display:none;">Loading model...</div>
 
   <script type="module">
-    import { pipeline } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.8.1';
-    
+    import { pipeline } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@4';
+
     let extractor;
-    
+
     // Initialize model on page load
     document.getElementById('loading').style.display = 'block';
     extractor = await pipeline(
@@ -41,18 +41,18 @@ All examples use the same task and model for consistency:
       'onnx-community/all-MiniLM-L6-v2-ONNX'
     );
     document.getElementById('loading').style.display = 'none';
-    
+
     window.generateEmbedding = async function() {
       const text = document.getElementById('input').value;
       const output = await extractor(text, { pooling: 'mean', normalize: true });
-      
+
       document.getElementById('result').innerHTML = `
         <h3>Embedding Generated:</h3>
         <p>Dimensions: ${output.data.length}</p>
         <p>First 5 values: ${Array.from(output.data).slice(0, 5).join(', ')}</p>
       `;
     };
-    
+
     // Cleanup on page unload
     window.addEventListener('beforeunload', () => {
       if (extractor) extractor.dispose();
@@ -104,19 +104,24 @@ All examples use the same task and model for consistency:
   </div>
 
   <script type="module">
-    import { pipeline } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.8.1';
-    
+    import { pipeline } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@4';
+
     let extractor;
     const fileProgressBars = {};
     const progressContainer = document.getElementById('progress-container');
-    
+
     extractor = await pipeline(
       'feature-extraction',
       'onnx-community/all-MiniLM-L6-v2-ONNX',
       {
         progress_callback: (info) => {
-          document.getElementById('status').textContent = `${info.status}: ${info.file}`;
-          
+          if (info.status === 'progress_total') {
+            document.getElementById('status').textContent = `Total: ${info.progress.toFixed(1)}%`;
+            return;
+          }
+
+          document.getElementById('status').textContent = `${info.status}: ${info.file ?? ''}`;
+
           if (info.status === 'progress') {
             // Create progress bar for each file
             if (!fileProgressBars[info.file]) {
@@ -131,11 +136,11 @@ All examples use the same task and model for consistency:
               progressContainer.appendChild(fileDiv);
               fileProgressBars[info.file] = fileDiv.querySelector('.progress-fill');
             }
-            
+
             // Update progress
             fileProgressBars[info.file].style.width = `${info.progress}%`;
           }
-          
+
           if (info.status === 'ready') {
             document.getElementById('loading').style.display = 'none';
             document.getElementById('app').style.display = 'block';
@@ -143,16 +148,16 @@ All examples use the same task and model for consistency:
         }
       }
     );
-    
+
     window.generateEmbedding = async function() {
       const text = document.getElementById('input').value;
       const output = await extractor(text, { pooling: 'mean', normalize: true });
-      
+
       document.getElementById('result').innerHTML = `
         <p>Embedding: ${output.data.length} dimensions</p>
       `;
     };
-    
+
     // Cleanup on page unload
     window.addEventListener('beforeunload', () => {
       if (extractor) extractor.dispose();
@@ -175,13 +180,13 @@ async function generateEmbedding(text) {
     'feature-extraction',
     'onnx-community/all-MiniLM-L6-v2-ONNX'
   );
-  
+
   const output = await extractor(text, { pooling: 'mean', normalize: true });
-  
+
   console.log('Text:', text);
   console.log('Embedding dimensions:', output.data.length);
   console.log('First 5 values:', Array.from(output.data).slice(0, 5));
-  
+
   await extractor.dispose();
 }
 
@@ -200,32 +205,32 @@ async function embedDocuments(documents) {
     'feature-extraction',
     'onnx-community/all-MiniLM-L6-v2-ONNX'
   );
-  
+
   console.log(`Processing ${documents.length} documents...`);
-  
+
   const embeddings = [];
-  
+
   for (let i = 0; i < documents.length; i++) {
-    const output = await extractor(documents[i], { 
-      pooling: 'mean', 
-      normalize: true 
+    const output = await extractor(documents[i], {
+      pooling: 'mean',
+      normalize: true
     });
-    
+
     embeddings.push({
       text: documents[i],
       embedding: Array.from(output.data)
     });
-    
+
     console.log(`Processed ${i + 1}/${documents.length}`);
   }
-  
+
   await fs.writeFile(
     'embeddings.json',
     JSON.stringify(embeddings, null, 2)
   );
-  
+
   console.log('Saved to embeddings.json');
-  
+
   await extractor.dispose();
 }
 
@@ -246,45 +251,50 @@ import { pipeline } from '@huggingface/transformers';
 
 async function main() {
   const text = process.argv[2] || 'Hello, world!';
-  
+
   console.log('Loading model...');
-  
+
   const fileProgress = {};
-  
+
   const extractor = await pipeline(
     'feature-extraction',
     'onnx-community/all-MiniLM-L6-v2-ONNX',
     {
       progress_callback: (info) => {
+        if (info.status === 'progress_total') {
+          process.stdout.write(`\r\x1b[KTotal: ${info.progress.toFixed(1)}%`);
+          return;
+        }
+
         if (info.status === 'progress') {
           fileProgress[info.file] = info.progress;
-          
+
           // Show all files progress
           const progressLines = Object.entries(fileProgress)
             .map(([file, progress]) => `  ${file}: ${progress.toFixed(1)}%`)
             .join('\n');
-          
+
           process.stdout.write(`\r\x1b[K${progressLines}`);
         }
-        
+
         if (info.status === 'done') {
           console.log(`\n✓ ${info.file} complete`);
         }
-        
+
         if (info.status === 'ready') {
           console.log('\nModel ready!');
         }
       }
     }
   );
-  
+
   console.log('Generating embedding...');
   const output = await extractor(text, { pooling: 'mean', normalize: true });
-  
+
   console.log(`\nText: "${text}"`);
   console.log(`Dimensions: ${output.data.length}`);
   console.log(`First 5 values: ${Array.from(output.data).slice(0, 5).join(', ')}`);
-  
+
   await extractor.dispose();
 }
 
@@ -308,9 +318,9 @@ export function EmbeddingGenerator() {
 
   const generate = async () => {
     if (!text) return;
-    
+
     setLoading(true);
-    
+
     // Load model on first generate
     if (!extractorRef.current) {
       extractorRef.current = await pipeline(
@@ -318,10 +328,10 @@ export function EmbeddingGenerator() {
         'onnx-community/all-MiniLM-L6-v2-ONNX'
       );
     }
-    
-    const output = await extractorRef.current(text, { 
-      pooling: 'mean', 
-      normalize: true 
+
+    const output = await extractorRef.current(text, {
+      pooling: 'mean',
+      normalize: true
     });
     setEmbedding(Array.from(output.data));
     setLoading(false);
@@ -339,18 +349,18 @@ export function EmbeddingGenerator() {
   return (
     <div>
       <h2>Text Embedding Generator</h2>
-      
+
       <textarea
         value={text}
         onChange={(e) => setText(e.target.value)}
         placeholder="Enter text"
         disabled={loading}
       />
-      
+
       <button onClick={generate} disabled={loading || !text}>
         {loading ? 'Processing...' : 'Generate Embedding'}
       </button>
-      
+
       {embedding && (
         <div>
           <h3>Result:</h3>
@@ -380,27 +390,32 @@ export function EmbeddingGeneratorWithProgress() {
 
   const generate = async () => {
     if (!text) return;
-    
+
     setLoading(true);
-    
+
     // Load model on first generate
     if (!extractorRef.current) {
       setStatus('Loading model...');
-      
+
       extractorRef.current = await pipeline(
         'feature-extraction',
         'onnx-community/all-MiniLM-L6-v2-ONNX',
         {
           progress_callback: (info) => {
-            setStatus(`${info.status}: ${info.file}`);
-            
+            if (info.status === 'progress_total') {
+              setStatus(`Total: ${info.progress.toFixed(1)}%`);
+              return;
+            }
+
+            setStatus(`${info.status}: ${info.file ?? ''}`);
+
             if (info.status === 'progress') {
               setFileProgress(prev => ({
                 ...prev,
                 [info.file]: info.progress
               }));
             }
-            
+
             if (info.status === 'ready') {
               setStatus('Model ready!');
             }
@@ -408,11 +423,11 @@ export function EmbeddingGeneratorWithProgress() {
         }
       );
     }
-    
+
     setStatus('Generating embedding...');
-    const output = await extractorRef.current(text, { 
-      pooling: 'mean', 
-      normalize: true 
+    const output = await extractorRef.current(text, {
+      pooling: 'mean',
+      normalize: true
     });
     setEmbedding(Array.from(output.data));
     setStatus('Complete!');
@@ -431,7 +446,7 @@ export function EmbeddingGeneratorWithProgress() {
   return (
     <div>
       <h2>Text Embedding Generator</h2>
-      
+
       {loading && Object.keys(fileProgress).length > 0 && (
         <div>
           <p>{status}</p>
@@ -439,31 +454,31 @@ export function EmbeddingGeneratorWithProgress() {
             <div key={file} style={{ margin: '10px 0' }}>
               <div style={{ fontSize: '12px', marginBottom: '5px' }}>{file}</div>
               <div style={{ width: '100%', height: '20px', background: '#f0f0f0', borderRadius: '5px', overflow: 'hidden' }}>
-                <div 
-                  style={{ 
-                    width: `${progress}%`, 
-                    height: '100%', 
+                <div
+                  style={{
+                    width: `${progress}%`,
+                    height: '100%',
                     background: '#4CAF50',
                     transition: 'width 0.3s'
-                  }} 
+                  }}
                 />
               </div>
             </div>
           ))}
         </div>
       )}
-      
+
       <textarea
         value={text}
         onChange={(e) => setText(e.target.value)}
         placeholder="Enter text"
         disabled={loading}
       />
-      
+
       <button onClick={generate} disabled={loading || !text}>
         {loading ? 'Processing...' : 'Generate Embedding'}
       </button>
-      
+
       {embedding && (
         <div>
           <h3>Result:</h3>
@@ -502,16 +517,16 @@ let extractor;
 app.post('/embed', async (req, res) => {
   try {
     const { text } = req.body;
-    
+
     if (!text) {
       return res.status(400).json({ error: 'Text is required' });
     }
-    
-    const output = await extractor(text, { 
-      pooling: 'mean', 
-      normalize: true 
+
+    const output = await extractor(text, {
+      pooling: 'mean',
+      normalize: true
     });
-    
+
     res.json({
       text,
       embedding: Array.from(output.data),
@@ -553,16 +568,16 @@ async function initialize() {
 app.post('/embed', async (req, res) => {
   try {
     const { text } = req.body;
-    
+
     if (!text) {
       return res.status(400).json({ error: 'Text is required' });
     }
-    
-    const output = await extractor(text, { 
-      pooling: 'mean', 
-      normalize: true 
+
+    const output = await extractor(text, {
+      pooling: 'mean',
+      normalize: true
     });
-    
+
     res.json({
       embedding: Array.from(output.data),
       dimensions: output.data.length
@@ -574,19 +589,19 @@ app.post('/embed', async (req, res) => {
 
 async function shutdown(signal) {
   console.log(`\n${signal} received. Shutting down...`);
-  
+
   if (server) {
     server.close(() => {
       console.log('HTTP server closed');
     });
   }
-  
+
   if (extractor) {
     console.log('Disposing model...');
     await extractor.dispose();
     console.log('Model disposed');
   }
-  
+
   process.exit(0);
 }
 

package/bundled-skills/transformers-js/references/MODEL_REGISTRY.md

@@ -0,0 +1,197 @@
+# ModelRegistry Reference
+
+In Transformers.js v4, `ModelRegistry` provides a preflight API for model assets. You can inspect required files, estimate total download size, check cache state, and clear cached artifacts before calling `pipeline()`.
+
+This is useful for production UX where you want to:
+- show accurate download estimates before loading,
+- support offline-first flows,
+- avoid surprise bandwidth usage,
+- and keep cache management explicit.
+
+## Table of Contents
+
+1. [Overview](#overview)
+2. [Core APIs](#core-apis)
+3. [Recommended Workflow](#recommended-workflow)
+4. [Examples](#examples)
+5. [Best Practices](#best-practices)
+
+## Overview
+
+```javascript
+import { ModelRegistry } from '@huggingface/transformers';
+```
+
+`ModelRegistry` works with the same task/model/options you pass to `pipeline()`.
+
+Typical tuple:
+
+```javascript
+const task = 'feature-extraction';
+const modelId = 'onnx-community/all-MiniLM-L6-v2-ONNX';
+const modelOptions = { dtype: 'fp32' };
+```
+
+## Core APIs
+
+### `get_pipeline_files(task, modelId, modelOptions)`
+
+Returns all files needed to initialize that pipeline configuration.
+
+```javascript
+const files = await ModelRegistry.get_pipeline_files(task, modelId, modelOptions);
+// Example: ['config.json', 'onnx/model.onnx', 'tokenizer.json', ...]
+```
+
+Use this to build preflight checks and download manifests.
+
+### `get_file_metadata(modelId, file)`
+
+Returns metadata for a single file (including size when available).
+
+```javascript
+const metadata = await ModelRegistry.get_file_metadata(modelId, 'onnx/model.onnx');
+console.log(metadata);
+```
+
+Use this to compute total transfer size and identify large artifacts.
+
+### `is_pipeline_cached(task, modelId, modelOptions)`
+
+Checks whether required files are already available in cache.
+
+```javascript
+const cached = await ModelRegistry.is_pipeline_cached(task, modelId, modelOptions);
+console.log(cached ? 'Ready offline' : 'Needs download');
+```
+
+Use this to gate offline mode and skip unnecessary preload steps.
+
+### `clear_pipeline_cache(task, modelId, modelOptions)`
+
+Clears cached assets for a specific pipeline tuple.
+
+```javascript
+await ModelRegistry.clear_pipeline_cache(task, modelId, modelOptions);
+```
+
+Use this for cache invalidation, testing, or space reclamation.
+
+### `get_available_dtypes(modelId)`
+
+Returns precision/quantization formats available for the model.
+
+```javascript
+const dtypes = await ModelRegistry.get_available_dtypes(modelId);
+// Example: ['fp32', 'fp16', 'q4', 'q4f16']
+```
+
+Use this to choose the best runtime profile (quality vs. speed vs. memory).
+
+## Recommended Workflow
+
+For robust loading UX:
+
+1. Resolve the exact task/model/options tuple.
+2. Call `get_pipeline_files(...)`.
+3. Fetch metadata per file and compute total size.
+4. Call `is_pipeline_cached(...)`.
+5. If not cached, show user-facing size/progress expectations.
+6. Load via `pipeline(...)` and use `progress_total` in `progress_callback`.
+
+```javascript
+import { ModelRegistry, pipeline } from '@huggingface/transformers';
+
+const task = 'feature-extraction';
+const modelId = 'onnx-community/all-MiniLM-L6-v2-ONNX';
+const modelOptions = { dtype: 'q8' };
+
+const files = await ModelRegistry.get_pipeline_files(task, modelId, modelOptions);
+
+const metadata = await Promise.all(
+  files.map((file) => ModelRegistry.get_file_metadata(modelId, file))
+);
+
+const totalBytes = metadata.reduce((sum, item) => sum + (item?.size ?? 0), 0);
+const totalMB = (totalBytes / 1024 / 1024).toFixed(2);
+
+const cached = await ModelRegistry.is_pipeline_cached(task, modelId, modelOptions);
+console.log({ fileCount: files.length, totalMB, cached });
+
+const pipe = await pipeline(task, modelId, {
+  ...modelOptions,
+  progress_callback: (info) => {
+    if (info.status === 'progress_total') {
+      console.log(`Loading: ${info.progress.toFixed(1)}%`);
+    }
+  },
+});
+
+await pipe.dispose();
+```
+
+## Examples
+
+### Example 1: Offer dtype choice dynamically
+
+```javascript
+import { ModelRegistry, pipeline } from '@huggingface/transformers';
+
+const task = 'text-generation';
+const modelId = 'onnx-community/Qwen2.5-0.5B-Instruct';
+
+const dtypes = await ModelRegistry.get_available_dtypes(modelId);
+const preferred = dtypes.includes('q4') ? 'q4' : dtypes[0] ?? 'fp32';
+
+const generator = await pipeline(task, modelId, { dtype: preferred });
+// ... inference
+await generator.dispose();
+```
+
+### Example 2: Only clear one pipeline cache entry
+
+```javascript
+import { ModelRegistry } from '@huggingface/transformers';
+
+await ModelRegistry.clear_pipeline_cache(
+  'feature-extraction',
+  'onnx-community/all-MiniLM-L6-v2-ONNX',
+  { dtype: 'fp32' }
+);
+```
+
+This avoids wiping unrelated model caches.
+
+### Example 3: Offline gate
+
+```javascript
+import { ModelRegistry, env, pipeline } from '@huggingface/transformers';
+
+const task = 'feature-extraction';
+const modelId = 'onnx-community/all-MiniLM-L6-v2-ONNX';
+const modelOptions = { dtype: 'q8' };
+
+const cached = await ModelRegistry.is_pipeline_cached(task, modelId, modelOptions);
+
+if (!cached) {
+  throw new Error('Model not cached yet. Connect once to download assets.');
+}
+
+env.allowRemoteModels = false;
+const pipe = await pipeline(task, modelId, { ...modelOptions, local_files_only: true });
+```
+
+## Best Practices
+
+1. Use `ModelRegistry` before `pipeline()` when you need predictable download UX.
+2. Cache decisions should be per task/model/options tuple (dtype and revision matter).
+3. Use `progress_total` for user-facing progress bars; keep per-file progress optional.
+4. Prefer selective invalidation with `clear_pipeline_cache(...)` over broad cache deletion.
+5. In offline mode, combine `is_pipeline_cached(...)` with `local_files_only: true` and `env.allowRemoteModels = false`.
+
+## Related Documentation
+
+- [Pipeline Options](./PIPELINE_OPTIONS.md) - `pipeline()` options and progress callback
+- [Configuration Reference](./CONFIGURATION.md) - `env` settings for local/remote loading
+- [Caching Reference](./CACHE.md) - Browser, filesystem, and custom cache behavior
+- [Main Skill Guide](../SKILL.md) - Practical usage patterns