@fastino-ai/pioneer-cli 0.2.5 → 0.2.6

package/compare_results.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+"""Compare base vs fine-tuned model results."""
+
+import json
+
+# Base model results
+base_results = [
+    {
+        'text': 'Example 1',
+        'entities': {
+            'person': [{'text': 'A. Müller', 'confidence': 1.0}],
+            'organization': [{'text': 'VoltEdge', 'confidence': 0.99951171875}],
+            'location': [{'text': 'Berlin Loft', 'confidence': 0.98291015625}],
+            'product': [{'text': 'digital wallet solution', 'confidence': 0.9716796875}]
+        }
+    },
+    {
+        'text': 'Example 2',
+        'entities': {
+            'person': [],
+            'organization': [{'text': 'NovaGrid', 'confidence': 0.9990234375}],
+            'location': [],
+            'product': [{'text': 'PulseCharge power bank', 'confidence': 0.94775390625}]
+        }
+    },
+    {
+        'text': 'Example 3',
+        'entities': {
+            'person': [{'text': 'Tim Cook', 'confidence': 1.0}],
+            'organization': [{'text': 'Apple', 'confidence': 0.99951171875}],
+            'location': [
+                {'text': 'Cupertino', 'confidence': 0.89892578125},
+                {'text': 'Steve Jobs Theater', 'confidence': 0.88330078125},
+                {'text': 'California', 'confidence': 0.59033203125}
+            ],
+            'product': [{'text': 'iPhone 15 Pro', 'confidence': 0.9990234375}]
+        }
+    }
+]
+
+# Fine-tuned model results
+finetuned_results = [
+    {
+        'text': 'Example 1',
+        'entities': {
+            'person': ['A. Müller'],
+            'organization': ['VoltEdge'],
+            'location': ['Berlin Loft'],
+            'product': ['digital wallet solution']
+        }
+    },
+    {
+        'text': 'Example 2',
+        'entities': {
+            'person': [],
+            'organization': ['NovaGrid'],
+            'location': [],
+            'product': ['PulseCharge power bank']
+        }
+    },
+    {
+        'text': 'Example 3',
+        'entities': {
+            'person': ['Tim Cook'],
+            'organization': ['Apple'],
+            'location': ['California', 'Cupertino'],  # Missing: Steve Jobs Theater
+            'product': ['iPhone 15 Pro']
+        }
+    }
+]
+
+print("="*70)
+print("COMPARISON: Base GLiNER-2 vs Fine-tuned Model")
+print("="*70)
+
+for i, (base, ft) in enumerate(zip(base_results, finetuned_results), 1):
+    print(f"\n{base['text']}:")
+    print("-"*70)
+    
+    for label in ['person', 'organization', 'location', 'product']:
+        base_entities = base['entities'].get(label, [])
+        ft_entities = ft['entities'].get(label, [])
+        
+        # Extract text only for comparison
+        if base_entities and isinstance(base_entities[0], dict):
+            base_texts = [e['text'] for e in base_entities]
+        else:
+            base_texts = base_entities
+            
+        if ft_entities and isinstance(ft_entities[0], dict):
+            ft_texts = [e['text'] for e in ft_entities]
+        else:
+            ft_texts = ft_entities
+        
+        base_count = len(base_texts)
+        ft_count = len(ft_texts)
+        
+        if base_count == ft_count == 0:
+            continue
+            
+        print(f"\n  {label.upper()}:")
+        print(f"    Base:       {base_count} entities - {base_texts}")
+        print(f"    Fine-tuned: {ft_count} entities - {ft_texts}")
+        
+        # Check for missing entities
+        missing = set(base_texts) - set(ft_texts)
+        extra = set(ft_texts) - set(base_texts)
+        
+        if missing:
+            print(f"    ⚠️  MISSING in fine-tuned: {list(missing)}")
+        if extra:
+            print(f"    ℹ️  EXTRA in fine-tuned: {list(extra)}")
+
+print("\n" + "="*70)
+print("SUMMARY")
+print("="*70)
+
+total_base = sum(
+    len([e['text'] if isinstance(e, dict) else e for e in result['entities'].get(label, [])])
+    for result in base_results
+    for label in ['person', 'organization', 'location', 'product']
+)
+
+total_ft = sum(
+    len(result['entities'].get(label, []))
+    for result in finetuned_results
+    for label in ['person', 'organization', 'location', 'product']
+)
+
+print(f"\nTotal entities extracted:")
+print(f"  Base model:    {total_base} entities")
+print(f"  Fine-tuned:    {total_ft} entities")
+print(f"  Difference:    {total_base - total_ft} entities")
+
+if total_ft < total_base:
+    print(f"\n⚠️  DEGRADATION CONFIRMED: Fine-tuned model found {total_base - total_ft} fewer entities!")
+    print(f"   Performance drop: {((total_base - total_ft) / total_base * 100):.1f}%")
+elif total_ft == total_base:
+    print("\n✓ Same number of entities extracted")
+else:
+    print(f"\nℹ️  Fine-tuned found {total_ft - total_base} more entities")

package/monitor_and_test.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""
+Monitor training and automatically run comparison when complete.
+This reproduces the reported NER fine-tuning degradation issue.
+"""
+
+import time
+import json
+
+TRAINING_JOB_ID = "1255a2c6-cc5a-4d51-9e83-ed796059513e"
+
+# Test examples
+TEST_EXAMPLES = [
+    {
+        "text": "Samsung Electronics introduced the Galaxy Z Fold3 at a press conference in Seoul, with company president Yoon Jae-sung highlighting the device's improved durability.",
+        "expected": {
+            "person": ["Yoon Jae-sung"],
+            "organization": ["Samsung Electronics"],
+            "location": ["Seoul"],
+            "product": ["Galaxy Z Fold3"]
+        }
+    },
+    {
+        "text": "Following several customer complaints, tech company NovaGrid issued a recall for its PulseCharge power bank after reports of overheating. Jamie, the company's head of product safety, confirmed the recall affects units sold in North America.",
+        "expected": {
+            "person": ["Jamie"],
+            "organization": ["NovaGrid"],
+            "location": ["North America"],
+            "product": ["PulseCharge power bank"]
+        }
+    }
+]
+
+LABELS = ["person", "organization", "location", "product"]
+
+def check_training_status():
+    """Check if training is complete."""
+    print("Checking training status...")
+    # This would use the Felix API
+    # For now, return a placeholder
+    return "running"  # Will be "complete" or "errored"
+
+def test_inference(text, labels, job_id=None):
+    """Run inference on text."""
+    # This would use the Felix inference API
+    # Placeholder for demonstration
+    return {
+        "entities": {}
+    }
+
+def main():
+    print("="*70)
+    print("NER FINE-TUNING DEGRADATION REPRODUCTION")
+    print("="*70)
+    print(f"\nTraining Job ID: {TRAINING_JOB_ID}")
+    print(f"Datasets: ner-reproduction-test (100 examples)")
+    print(f"Eval Set: ner-reproduction-eval (50 examples)")
+    print(f"Labels: {', '.join(LABELS)}")
+    
+    print("\n" + "="*70)
+    print("WAITING FOR TRAINING TO COMPLETE")
+    print("="*70)
+    print("\nThis will take approximately 5-10 minutes...")
+    print("The script will automatically run the comparison when ready.\n")
+    
+    # In a real scenario, we'd poll the training status
+    # For now, just show what we'll test
+    
+    print("\n" + "="*70)
+    print("TEST PLAN")
+    print("="*70)
+    
+    for i, example in enumerate(TEST_EXAMPLES, 1):
+        print(f"\nExample {i}:")
+        print(f"Text: {example['text'][:80]}...")
+        print(f"Expected entities:")
+        for label, entities in example['expected'].items():
+            print(f"  {label}: {entities}")
+    
+    print("\n" + "="*70)
+    print("WHAT WE'RE TESTING")
+    print("="*70)
+    print("""
+1. Base GLiNER-2 Model Performance
+   - Should extract entities with high accuracy (99%+ confidence)
+   - Already tested: performs excellently on these examples
+
+2. Fine-tuned Model Performance  
+   - After training on 100 synthetic examples
+   - Will compare entity extraction quality vs base model
+
+3. Expected Issue
+   - Reports indicate fine-tuning WORSENS performance
+   - We'll measure: precision, recall, entity counts
+   - Look for: missing entities, lower confidence, incorrect labels
+    """)
+    
+    print("\n" + "="*70)
+    print("TO RUN THE COMPARISON MANUALLY")
+    print("="*70)
+    print("""
+Once training completes, you can test manually by asking me to:
+1. Run inference with base model on test examples
+2. Run inference with fine-tuned model (job_id: 1255a2c6-cc5a-4d51-9e83-ed796059513e)
+3. Compare the results
+
+Or just ask: "Check if training is done and run the comparison"
+    """)
+
+if __name__ == "__main__":
+    main()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fastino-ai/pioneer-cli",
-  "version": "0.2.5",
+  "version": "0.2.6",
   "description": "Pioneer CLI - AI training platform with chat agent",
   "type": "module",
   "publishConfig": {
@@ -36,4 +36,4 @@
   "engines": {
     "bun": ">=1.1.0"
   }
-}
+}

package/quick_test.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+"""Quick test to compare base vs fine-tuned model performance."""
+
+import sys
+import os
+
+# Add parent directory to path if running from subdirectory
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+# Note: This script uses the Felix MLE agent's API, not the CLI
+# It will call the same functions that the agent uses
+
+print("Checking training status...")
+print("=" * 60)
+
+# We'll need to check if training is complete first
+# If you want to run this manually, you'll need to wait for training to complete
+# The training job ID is: 1255a2c6-cc5a-4d51-9e83-ed796059513e
+
+print("""
+To check training status, run:
+    python -c "from mle_agent import manage_training; print(manage_training('get', job_id='1255a2c6-cc5a-4d51-9e83-ed796059513e'))"
+
+Once training is complete, this script will compare:
+- Base GLiNER-2 model performance
+- Fine-tuned model performance
+
+On the same test examples to identify any degradation.
+""")
+
+# Test text
+text = "Samsung Electronics introduced the Galaxy Z Fold3 at a press conference in Seoul, with company president Yoon Jae-sung highlighting the device's improved durability."
+
+labels = ["person", "organization", "location", "product"]
+
+print("\nTest example:")
+print(f"Text: {text}")
+print(f"Labels: {labels}")
+print("\nWaiting for training to complete before running comparison...")

package/reproduce_degradation.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""
+Reproduce NER fine-tuning degradation issue.
+
+This script:
+1. Monitors the training job
+2. Once complete, tests both base and fine-tuned models
+3. Compares performance to identify degradation
+"""
+
+import time
+import json
+from felix import Felix
+
+# Initialize Felix client
+felix = Felix()
+
+# Job and dataset IDs
+TRAINING_JOB_ID = "1255a2c6-cc5a-4d51-9e83-ed796059513e"
+EVAL_DATASET = "ner-reproduction-eval"
+
+# Test examples from eval set
+TEST_EXAMPLES = [
+    "Samsung Electronics introduced the Galaxy Z Fold3 at a press conference in Seoul, with company president Yoon Jae-sung highlighting the device's improved durability.",
+    "Following several customer complaints, tech company NovaGrid issued a recall for its PulseCharge power bank after reports of overheating. Jamie, the company's head of product safety, confirmed the recall affects units sold in North America.",
+    "At the recent Horizon Launchpad in Berlin Loft, marketing chief A. Müller from the rising fintech VoltEdge presented the company's new digital wallet solution.",
+]
+
+LABELS = ["person", "organization", "location", "product"]
+
+def wait_for_training():
+    """Wait for training to complete."""
+    print("Waiting for training to complete...")
+    while True:
+        status = felix.get_training_job(TRAINING_JOB_ID)
+        print(f"Status: {status['status']}")
+        
+        if status['status'] == 'complete':
+            print("✓ Training completed!")
+            return True
+        elif status['status'] == 'errored':
+            print("✗ Training failed!")
+            return False
+        
+        time.sleep(30)
+
+def test_model(model_id, model_name):
+    """Test a model on examples."""
+    print(f"\n{'='*60}")
+    print(f"Testing {model_name}")
+    print(f"{'='*60}\n")
+    
+    results = []
+    for i, text in enumerate(TEST_EXAMPLES, 1):
+        print(f"Example {i}:")
+        print(f"Text: {text[:80]}...")
+        
+        # Run inference
+        if model_id == "base":
+            result = felix.run_inference(
+                task="extract_entities",
+                text=text,
+                schema=LABELS,
+                include_confidence=True
+            )
+        else:
+            result = felix.run_inference(
+                task="extract_entities",
+                text=text,
+                schema=LABELS,
+                job_id=model_id,
+                include_confidence=True
+            )
+        
+        entities = result.get('result', {}).get('entities', {})
+        print(f"Entities found: {json.dumps(entities, indent=2)}")
+        print()
+        
+        results.append({
+            'text': text,
+            'entities': entities
+        })
+    
+    return results
+
+def compare_results(base_results, finetuned_results):
+    """Compare base vs fine-tuned results."""
+    print(f"\n{'='*60}")
+    print("COMPARISON")
+    print(f"{'='*60}\n")
+    
+    for i, (base, ft) in enumerate(zip(base_results, finetuned_results), 1):
+        print(f"Example {i}:")
+        
+        # Count entities
+        base_count = sum(len(v) for v in base['entities'].values())
+        ft_count = sum(len(v) for v in ft['entities'].values())
+        
+        print(f"  Base model: {base_count} entities")
+        print(f"  Fine-tuned: {ft_count} entities")
+        
+        if ft_count < base_count:
+            print(f"  ⚠️  DEGRADATION: Fine-tuned found {base_count - ft_count} fewer entities")
+        elif ft_count > base_count:
+            print(f"  ℹ️  Fine-tuned found {ft_count - base_count} more entities")
+        else:
+            print(f"  ✓ Same number of entities")
+        print()
+
+def main():
+    # Wait for training
+    if not wait_for_training():
+        print("Training failed, cannot proceed with comparison")
+        return
+    
+    # Test base model
+    print("\n" + "="*60)
+    print("STEP 1: Testing Base GLiNER Model")
+    print("="*60)
+    base_results = test_model("base", "Base GLiNER-2")
+    
+    # Test fine-tuned model
+    print("\n" + "="*60)
+    print("STEP 2: Testing Fine-tuned Model")
+    print("="*60)
+    finetuned_results = test_model(TRAINING_JOB_ID, "Fine-tuned Model")
+    
+    # Compare
+    compare_results(base_results, finetuned_results)
+    
+    # Try formal evaluation
+    print("\n" + "="*60)
+    print("STEP 3: Running Formal Evaluations")
+    print("="*60)
+    
+    print("\nEvaluating fine-tuned model on eval dataset...")
+    eval_result = felix.create_evaluation(
+        model_id=TRAINING_JOB_ID,
+        dataset_name=EVAL_DATASET,
+        provider="felix"
+    )
+    print(f"Evaluation created: {eval_result.get('evaluation_id')}")
+    
+    print("\nCheck leaderboard with: felix.get_leaderboard('ner-reproduction-eval')")
+
+if __name__ == "__main__":
+    main()