agentic-flow 1.9.0 → 1.9.1

package/agentic-llm/benchmarks/finetune_comparison.py

@@ -1,252 +0,0 @@
-#!/usr/bin/env python3
-"""
-Before/After Fine-Tuning Comparison Benchmarks
-Measures improvement from MCP tool fine-tuning
-"""
-
-import json
-import time
-from pathlib import Path
-from typing import Dict, List
-import logging
-import sys
-
-sys.path.append(str(Path(__file__).parent.parent))
-from validation.mcp_validator import MCPToolValidator
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-
-class FineTuneComparison:
-    """Compare baseline vs fine-tuned model"""
-
-    def __init__(self, output_dir: str = "/app/benchmarks/finetune_results"):
-        self.output_dir = Path(output_dir)
-        self.output_dir.mkdir(parents=True, exist_ok=True)
-
-    def get_test_cases(self) -> List[Dict]:
-        """Get comprehensive test cases for MCP tools"""
-        return [
-            # READ tool
-            {"query": "Read the package.json file", "expected_tool": "read", "expected_params": {"file_path": "package.json"}},
-            {"query": "Show me the README", "expected_tool": "read", "expected_params": {"file_path": "README.md"}},
-            {"query": "Display the config file", "expected_tool": "read", "expected_params": {}},
-
-            # WRITE tool
-            {"query": "Create a new utils.ts file", "expected_tool": "write", "expected_params": {"file_path": "utils.ts"}},
-            {"query": "Make a server.js with Express", "expected_tool": "write", "expected_params": {"file_path": "server.js"}},
-
-            # EDIT tool
-            {"query": "Change the port from 3000 to 8080", "expected_tool": "edit", "expected_params": {}},
-            {"query": "Rename getData to fetchData", "expected_tool": "edit", "expected_params": {}},
-
-            # BASH tool
-            {"query": "Run the tests", "expected_tool": "bash", "expected_params": {"command": "npm test"}},
-            {"query": "Install dependencies", "expected_tool": "bash", "expected_params": {"command": "npm install"}},
-            {"query": "Build the project", "expected_tool": "bash", "expected_params": {}},
-            {"query": "Start the development server", "expected_tool": "bash", "expected_params": {}},
-
-            # GREP tool
-            {"query": "Search for TODO comments", "expected_tool": "grep", "expected_params": {"pattern": "TODO"}},
-            {"query": "Find all authentication functions", "expected_tool": "grep", "expected_params": {}},
-            {"query": "Look for error handling code", "expected_tool": "grep", "expected_params": {}},
-
-            # GLOB tool
-            {"query": "List all TypeScript files", "expected_tool": "glob", "expected_params": {"pattern": "**/*.ts"}},
-            {"query": "Show all test files", "expected_tool": "glob", "expected_params": {}},
-            {"query": "Find all components", "expected_tool": "glob", "expected_params": {}},
-
-            # Complex queries
-            {"query": "What's in the authentication config", "expected_tool": "read", "expected_params": {}},
-            {"query": "Execute the linter", "expected_tool": "bash", "expected_params": {}},
-            {"query": "Locate API endpoint definitions", "expected_tool": "grep", "expected_params": {}},
-            {"query": "Get all JavaScript files in src", "expected_tool": "glob", "expected_params": {}},
-        ]
-
-    def benchmark_model(
-        self,
-        model_path: str,
-        model_name: str,
-        test_cases: List[Dict]
-    ) -> Dict:
-        """Benchmark a model on MCP tool calling"""
-        logger.info(f"\nBenchmarking: {model_name}")
-        logger.info("=" * 60)
-
-        try:
-            # Create validator
-            validator = MCPToolValidator(model_path=model_path)
-
-            # Run validation
-            start_time = time.time()
-            results = validator.validate_tool_accuracy(test_cases)
-            duration = time.time() - start_time
-
-            # Add metadata
-            results["model_name"] = model_name
-            results["model_path"] = model_path
-            results["duration_seconds"] = duration
-            results["avg_time_per_test"] = duration / len(test_cases)
-
-            return results
-
-        except Exception as e:
-            logger.error(f"Error benchmarking {model_name}: {e}")
-            return {
-                "model_name": model_name,
-                "error": str(e),
-                "total": len(test_cases),
-                "accuracy": 0.0
-            }
-
-    def calculate_improvements(
-        self,
-        baseline: Dict,
-        finetuned: Dict
-    ) -> Dict:
-        """Calculate improvement metrics"""
-        improvements = {
-            "accuracy_improvement": finetuned["accuracy"] - baseline["accuracy"],
-            "accuracy_improvement_pct": (
-                (finetuned["accuracy"] - baseline["accuracy"]) / baseline["accuracy"] * 100
-                if baseline["accuracy"] > 0 else 0
-            ),
-            "param_accuracy_improvement": finetuned["param_accuracy"] - baseline["param_accuracy"],
-            "tool_detection_improvement": (
-                (finetuned["tool_found"] / finetuned["total"]) -
-                (baseline["tool_found"] / baseline["total"])
-            ) * 100,
-            "meets_target": finetuned["accuracy"] >= 95.0,
-            "quality_tier": self._get_quality_tier(finetuned["accuracy"])
-        }
-
-        return improvements
-
-    def _get_quality_tier(self, accuracy: float) -> str:
-        """Get quality tier based on accuracy"""
-        if accuracy >= 95:
-            return "EXCELLENT - Production Ready"
-        elif accuracy >= 85:
-            return "GOOD - Acceptable for most use cases"
-        elif accuracy >= 75:
-            return "FAIR - Needs improvement"
-        else:
-            return "POOR - Significant training needed"
-
-    def run_comparison(
-        self,
-        baseline_path: str = None,
-        finetuned_path: str = "/app/checkpoints/mcp_finetuned/final_model"
-    ) -> Dict:
-        """Run complete before/after comparison"""
-        logger.info("\n" + "=" * 60)
-        logger.info("MCP FINE-TUNING COMPARISON BENCHMARK")
-        logger.info("=" * 60)
-
-        # Get test cases
-        test_cases = self.get_test_cases()
-        logger.info(f"\nTest cases: {len(test_cases)}")
-
-        # Benchmark baseline (if provided)
-        baseline_results = None
-        if baseline_path:
-            baseline_results = self.benchmark_model(
-                model_path=baseline_path,
-                model_name="Baseline (Pre-fine-tuning)",
-                test_cases=test_cases
-            )
-
-        # Benchmark fine-tuned model
-        finetuned_results = self.benchmark_model(
-            model_path=finetuned_path,
-            model_name="Fine-tuned (MCP Tools)",
-            test_cases=test_cases
-        )
-
-        # Calculate improvements
-        improvements = None
-        if baseline_results:
-            improvements = self.calculate_improvements(baseline_results, finetuned_results)
-
-        # Create report
-        report = {
-            "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
-            "test_cases": len(test_cases),
-            "baseline": baseline_results,
-            "finetuned": finetuned_results,
-            "improvements": improvements
-        }
-
-        # Print summary
-        self._print_summary(report)
-
-        # Save report
-        report_path = self.output_dir / "finetune_comparison.json"
-        with open(report_path, "w") as f:
-            json.dump(report, f, indent=2)
-
-        logger.info(f"\n📄 Report saved to: {report_path}")
-
-        return report
-
-    def _print_summary(self, report: Dict):
-        """Print comparison summary"""
-        logger.info("\n" + "=" * 60)
-        logger.info("RESULTS SUMMARY")
-        logger.info("=" * 60)
-
-        baseline = report.get("baseline")
-        finetuned = report["finetuned"]
-        improvements = report.get("improvements")
-
-        # Fine-tuned results
-        logger.info(f"\n🎯 Fine-tuned Model:")
-        logger.info(f"  Tool Accuracy: {finetuned['accuracy']:.1f}%")
-        logger.info(f"  Parameter Accuracy: {finetuned['param_accuracy']:.1f}%")
-        logger.info(f"  Tool Detection: {finetuned['tool_found']}/{finetuned['total']}")
-        logger.info(f"  Quality Tier: {self._get_quality_tier(finetuned['accuracy'])}")
-
-        # Improvements
-        if improvements:
-            logger.info(f"\n📈 Improvements:")
-            logger.info(f"  Accuracy Gain: +{improvements['accuracy_improvement']:.1f}% ({improvements['accuracy_improvement_pct']:.1f}% improvement)")
-            logger.info(f"  Tool Detection: +{improvements['tool_detection_improvement']:.1f}%")
-            logger.info(f"  Meets 95% Target: {'✅ YES' if improvements['meets_target'] else '❌ NO'}")
-
-        # Recommendation
-        logger.info(f"\n💡 Recommendation:")
-        if finetuned['accuracy'] >= 95:
-            logger.info("  ✅ Model is production-ready for Claude Agent SDK")
-            logger.info("  ✅ Proceed with ONNX quantization and deployment")
-        elif finetuned['accuracy'] >= 85:
-            logger.info("  ✓ Model performs well for most use cases")
-            logger.info("  → Consider additional fine-tuning for critical applications")
-        else:
-            logger.info("  ⚠ Model needs improvement")
-            logger.info("  → Recommended: More training data or additional epochs")
-
-
-def main():
-    """Main entry point"""
-    import argparse
-
-    parser = argparse.ArgumentParser(description="Compare baseline vs fine-tuned model")
-    parser.add_argument("--baseline", type=str, help="Path to baseline model (optional)")
-    parser.add_argument("--finetuned", type=str, default="/app/checkpoints/mcp_finetuned/final_model", help="Path to fine-tuned model")
-    parser.add_argument("--output-dir", type=str, default="/app/benchmarks/finetune_results", help="Output directory")
-
-    args = parser.parse_args()
-
-    # Run comparison
-    comparison = FineTuneComparison(output_dir=args.output_dir)
-    report = comparison.run_comparison(
-        baseline_path=args.baseline,
-        finetuned_path=args.finetuned
-    )
-
-    logger.info("\n✅ Comparison complete!")
-
-
-if __name__ == "__main__":
-    main()

package/agentic-llm/benchmarks/results/.gitkeep

@@ -1,2 +0,0 @@
-# Benchmark results directory
-# All training results will be stored here

package/agentic-llm/benchmarks/results/README.md

@@ -1,59 +0,0 @@
-# Benchmark Results Directory
-
-This directory contains before/after benchmarks for Phi-4 fine-tuning.
-
-## Files Generated
-
-### Baseline (Before Fine-tuning)
-- `baseline_results.json` - Baseline model performance on MCP tools
-  - Tool detection rate
-  - Tool accuracy
-  - Parameter accuracy
-  - Average inference time
-
-### Post-Training (After Fine-tuning)
-- `finetune_comparison.json` - Comparison between baseline and fine-tuned
-  - Accuracy improvements
-  - Performance metrics
-  - Tool-by-tool breakdown
-
-### Final Report
-- `final_training_report.json` - Complete training summary
-  - Training duration
-  - Success criteria validation
-  - Improvement metrics
-- `TRAINING_REPORT.md` - Human-readable report
-
-## Benchmark Methodology
-
-### Test Set
-20 validation examples covering:
-- Basic tool calls (read, write, edit)
-- Complex queries (grep, glob, bash)
-- Multi-step workflows
-- Edge cases
-
-### Metrics Tracked
-
-**Accuracy Metrics:**
-- Tool Detection Rate: % queries where tool call was detected
-- Tool Correctness: % queries with correct tool selected
-- Parameter Accuracy: % queries with correct parameters
-
-**Performance Metrics:**
-- Inference latency (ms)
-- Tokens per second
-- GPU memory usage
-
-**Quality Tiers:**
-- EXCELLENT: >95% accuracy
-- GOOD: 85-95% accuracy
-- NEEDS_IMPROVEMENT: <85% accuracy
-
-## Success Criteria
-
-✅ **Production Ready** if:
-- Tool accuracy > 95%
-- Parameter accuracy > 90%
-- Tool detection > 97%
-- Inference latency < 200ms

package/agentic-llm/benchmarks/results/agent-booster-results.json

@@ -1,116 +0,0 @@
-{
-  "metadata": {
-    "timestamp": "2025-10-07T21:37:30.120Z",
-    "runtime": "simulated",
-    "parser": "typescript-simulation",
-    "dataset": "small-test-dataset.json",
-    "samples": 12
-  },
-  "samples": [
-    {
-      "id": "test-001",
-      "description": "Add type annotations to function",
-      "latency_ms": 93,
-      "success": true,
-      "confidence": 0.9449822564157511,
-      "strategy": "fuzzy_replace"
-    },
-    {
-      "id": "test-002",
-      "description": "Add error handling to async function",
-      "latency_ms": 84,
-      "success": true,
-      "confidence": 0.9339389891105419,
-      "strategy": "fuzzy_replace"
-    },
-    {
-      "id": "test-003",
-      "description": "Convert var to const/let",
-      "latency_ms": 93,
-      "success": true,
-      "confidence": 0.9101958963966182,
-      "strategy": "fuzzy_replace"
-    },
-    {
-      "id": "test-004",
-      "description": "Add JSDoc comments",
-      "latency_ms": 83,
-      "success": true,
-      "confidence": 0.9381150179825795,
-      "strategy": "fuzzy_replace"
-    },
-    {
-      "id": "test-005",
-      "description": "Convert callback to Promise",
-      "latency_ms": 89,
-      "success": true,
-      "confidence": 0.9229781191229839,
-      "strategy": "fuzzy_replace"
-    },
-    {
-      "id": "test-006",
-      "description": "Add null checks",
-      "latency_ms": 96,
-      "success": true,
-      "confidence": 0.8724959526639541,
-      "strategy": "fuzzy_replace"
-    },
-    {
-      "id": "test-007",
-      "description": "Convert to arrow function",
-      "latency_ms": 114,
-      "success": true,
-      "confidence": 0.8741935557077134,
-      "strategy": "fuzzy_replace"
-    },
-    {
-      "id": "test-008",
-      "description": "Add input validation",
-      "latency_ms": 116,
-      "success": true,
-      "confidence": 0.9069102302845496,
-      "strategy": "fuzzy_replace"
-    },
-    {
-      "id": "test-009",
-      "description": "Convert class to TypeScript",
-      "latency_ms": 115,
-      "success": true,
-      "confidence": 0.908663172419369,
-      "strategy": "fuzzy_replace"
-    },
-    {
-      "id": "test-010",
-      "description": "Add destructuring",
-      "latency_ms": 93,
-      "success": true,
-      "confidence": 0.8741994099440088,
-      "strategy": "fuzzy_replace"
-    },
-    {
-      "id": "test-011",
-      "description": "Add try-catch wrapper",
-      "latency_ms": 92,
-      "success": true,
-      "confidence": 0.9131141055251639,
-      "strategy": "fuzzy_replace"
-    },
-    {
-      "id": "test-012",
-      "description": "Add async/await",
-      "latency_ms": 118,
-      "success": true,
-      "confidence": 0.9356269250648608,
-      "strategy": "fuzzy_replace"
-    }
-  ],
-  "aggregate": {
-    "totalLatency": 1186,
-    "avgLatency": 98.83333333333333,
-    "p50Latency": 93,
-    "p95Latency": 118,
-    "p99Latency": 118,
-    "successRate": 100,
-    "totalCost": 0
-  }
-}

package/agentic-llm/benchmarks/results/morph-baseline-results.json

@@ -1,168 +0,0 @@
-{
-  "timestamp": "2025-10-07T21:08:37.134Z",
-  "model": "morph-v3-fast",
-  "total_tests": 12,
-  "successful_tests": 12,
-  "failed_tests": 0,
-  "performance": {
-    "total_time_ms": 16238,
-    "avg_latency_ms": 352,
-    "total_api_time_ms": 4219
-  },
-  "usage": {
-    "applies_used": 12,
-    "input_tokens": 963,
-    "output_tokens": 427,
-    "remaining_budget": {
-      "applies": 488,
-      "input_tokens": 299037,
-      "output_tokens": 99573
-    }
-  },
-  "results": [
-    {
-      "test_id": "test-001",
-      "description": "Add type annotations to function",
-      "category": "typescript-conversion",
-      "success": true,
-      "output": "function add(a: number, b: number): number {\n  return a + b;\n}\n\n",
-      "latency": 466,
-      "tokens": {
-        "input": 70,
-        "output": 20
-      }
-    },
-    {
-      "test_id": "test-002",
-      "description": "Add error handling to async function",
-      "category": "error-handling",
-      "success": true,
-      "output": "async function fetchData(url) {\n  const response = await fetch(url);\n  return response.json();\n}\n\n",
-      "latency": 252,
-      "tokens": {
-        "input": 77,
-        "output": 21
-      }
-    },
-    {
-      "test_id": "test-003",
-      "description": "Convert var to const/let",
-      "category": "modernization",
-      "success": true,
-      "output": "const count = 0;\nconst name = 'John';\nfor (let i = 0; i < 10; i++) {\n  count++;\n}\n\n",
-      "latency": 285,
-      "tokens": {
-        "input": 90,
-        "output": 34
-      }
-    },
-    {
-      "test_id": "test-004",
-      "description": "Add JSDoc comments",
-      "category": "documentation",
-      "success": true,
-      "output": "/**\n * Multiply two numbers.\n * @param {number} x - The first number.\n * @param {number} y - The second number.\n * @returns {number} The product of x and y.\n */\nfunction multiply(x, y) {\n  return x * y;\n}\n\n",
-      "latency": 541,
-      "tokens": {
-        "input": 70,
-        "output": 59
-      }
-    },
-    {
-      "test_id": "test-005",
-      "description": "Convert callback to Promise",
-      "category": "async-conversion",
-      "success": true,
-      "output": "function readFile(path) {\n  return new Promise((resolve, reject) => {\n    fs.readFile(path, 'utf8', (err, data) => {\n      if (err) reject(err);\n      else resolve(data);\n    });\n  });\n}",
-      "latency": 395,
-      "tokens": {
-        "input": 98,
-        "output": 51
-      }
-    },
-    {
-      "test_id": "test-006",
-      "description": "Add null checks",
-      "category": "safety",
-      "success": true,
-      "output": "function getUserName(user) {\n  return user.profile.name;\n}\n\n",
-      "latency": 298,
-      "tokens": {
-        "input": 67,
-        "output": 14
-      }
-    },
-    {
-      "test_id": "test-007",
-      "description": "Convert to arrow function",
-      "category": "modernization",
-      "success": true,
-      "output": "const numbers = [1, 2, 3, 4, 5];\nconst doubled = numbers.map(n => n * 2);\n\n",
-      "latency": 292,
-      "tokens": {
-        "input": 90,
-        "output": 31
-      }
-    },
-    {
-      "test_id": "test-008",
-      "description": "Add input validation",
-      "category": "validation",
-      "success": true,
-      "output": "function divide(a, b) {\n  if (typeof a !== 'number' || typeof b !== 'number') {\n    throw new Error('Both arguments must be numbers');\n  }\n  return a / b;\n}",
-      "latency": 394,
-      "tokens": {
-        "input": 68,
-        "output": 45
-      }
-    },
-    {
-      "test_id": "test-009",
-      "description": "Convert class to TypeScript",
-      "category": "typescript-conversion",
-      "success": true,
-      "output": "class Calculator {\n  constructor() {\n    this.result = 0;\n  }\n  add(value) {\n    this.result += value;\n  }\n}\n\n",
-      "latency": 269,
-      "tokens": {
-        "input": 85,
-        "output": 31
-      }
-    },
-    {
-      "test_id": "test-010",
-      "description": "Add destructuring",
-      "category": "modernization",
-      "success": true,
-      "output": "function formatUser(user) {\n  const { name, email, age } = user;\n  return `${name} (${email}) - ${age} years old`;\n}\n\n",
-      "latency": 319,
-      "tokens": {
-        "input": 97,
-        "output": 35
-      }
-    },
-    {
-      "test_id": "test-011",
-      "description": "Add try-catch wrapper",
-      "category": "error-handling",
-      "success": true,
-      "output": "function parseJSON(str) {\n  try {\n    return JSON.parse(str);\n  } catch (e) {\n    return null;\n  }\n}",
-      "latency": 331,
-      "tokens": {
-        "input": 69,
-        "output": 30
-      }
-    },
-    {
-      "test_id": "test-012",
-      "description": "Add async/await",
-      "category": "async-conversion",
-      "success": true,
-      "output": "function getData() {\n  return fetch('/api/data')\n    .then(res => res.json())\n    .then(data => data);\n}\n\nasync function getData() {\n  const res = await fetch('/api/data');\n  const data = await res.json();\n  return data;\n}",
-      "latency": 377,
-      "tokens": {
-        "input": 82,
-        "output": 56
-      }
-    }
-  ]
-}