agentic-flow 1.8.15 → 1.9.0

package/agentic-llm/benchmarks/finetune_comparison.py

@@ -0,0 +1,252 @@
+#!/usr/bin/env python3
+"""
+Before/After Fine-Tuning Comparison Benchmarks
+Measures improvement from MCP tool fine-tuning
+"""
+
+import json
+import time
+from pathlib import Path
+from typing import Dict, List
+import logging
+import sys
+
+sys.path.append(str(Path(__file__).parent.parent))
+from validation.mcp_validator import MCPToolValidator
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class FineTuneComparison:
+    """Compare baseline vs fine-tuned model"""
+
+    def __init__(self, output_dir: str = "/app/benchmarks/finetune_results"):
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+    def get_test_cases(self) -> List[Dict]:
+        """Get comprehensive test cases for MCP tools"""
+        return [
+            # READ tool
+            {"query": "Read the package.json file", "expected_tool": "read", "expected_params": {"file_path": "package.json"}},
+            {"query": "Show me the README", "expected_tool": "read", "expected_params": {"file_path": "README.md"}},
+            {"query": "Display the config file", "expected_tool": "read", "expected_params": {}},
+
+            # WRITE tool
+            {"query": "Create a new utils.ts file", "expected_tool": "write", "expected_params": {"file_path": "utils.ts"}},
+            {"query": "Make a server.js with Express", "expected_tool": "write", "expected_params": {"file_path": "server.js"}},
+
+            # EDIT tool
+            {"query": "Change the port from 3000 to 8080", "expected_tool": "edit", "expected_params": {}},
+            {"query": "Rename getData to fetchData", "expected_tool": "edit", "expected_params": {}},
+
+            # BASH tool
+            {"query": "Run the tests", "expected_tool": "bash", "expected_params": {"command": "npm test"}},
+            {"query": "Install dependencies", "expected_tool": "bash", "expected_params": {"command": "npm install"}},
+            {"query": "Build the project", "expected_tool": "bash", "expected_params": {}},
+            {"query": "Start the development server", "expected_tool": "bash", "expected_params": {}},
+
+            # GREP tool
+            {"query": "Search for TODO comments", "expected_tool": "grep", "expected_params": {"pattern": "TODO"}},
+            {"query": "Find all authentication functions", "expected_tool": "grep", "expected_params": {}},
+            {"query": "Look for error handling code", "expected_tool": "grep", "expected_params": {}},
+
+            # GLOB tool
+            {"query": "List all TypeScript files", "expected_tool": "glob", "expected_params": {"pattern": "**/*.ts"}},
+            {"query": "Show all test files", "expected_tool": "glob", "expected_params": {}},
+            {"query": "Find all components", "expected_tool": "glob", "expected_params": {}},
+
+            # Complex queries
+            {"query": "What's in the authentication config", "expected_tool": "read", "expected_params": {}},
+            {"query": "Execute the linter", "expected_tool": "bash", "expected_params": {}},
+            {"query": "Locate API endpoint definitions", "expected_tool": "grep", "expected_params": {}},
+            {"query": "Get all JavaScript files in src", "expected_tool": "glob", "expected_params": {}},
+        ]
+
+    def benchmark_model(
+        self,
+        model_path: str,
+        model_name: str,
+        test_cases: List[Dict]
+    ) -> Dict:
+        """Benchmark a model on MCP tool calling"""
+        logger.info(f"\nBenchmarking: {model_name}")
+        logger.info("=" * 60)
+
+        try:
+            # Create validator
+            validator = MCPToolValidator(model_path=model_path)
+
+            # Run validation
+            start_time = time.time()
+            results = validator.validate_tool_accuracy(test_cases)
+            duration = time.time() - start_time
+
+            # Add metadata
+            results["model_name"] = model_name
+            results["model_path"] = model_path
+            results["duration_seconds"] = duration
+            results["avg_time_per_test"] = duration / len(test_cases)
+
+            return results
+
+        except Exception as e:
+            logger.error(f"Error benchmarking {model_name}: {e}")
+            return {
+                "model_name": model_name,
+                "error": str(e),
+                "total": len(test_cases),
+                "accuracy": 0.0
+            }
+
+    def calculate_improvements(
+        self,
+        baseline: Dict,
+        finetuned: Dict
+    ) -> Dict:
+        """Calculate improvement metrics"""
+        improvements = {
+            "accuracy_improvement": finetuned["accuracy"] - baseline["accuracy"],
+            "accuracy_improvement_pct": (
+                (finetuned["accuracy"] - baseline["accuracy"]) / baseline["accuracy"] * 100
+                if baseline["accuracy"] > 0 else 0
+            ),
+            "param_accuracy_improvement": finetuned["param_accuracy"] - baseline["param_accuracy"],
+            "tool_detection_improvement": (
+                (finetuned["tool_found"] / finetuned["total"]) -
+                (baseline["tool_found"] / baseline["total"])
+            ) * 100,
+            "meets_target": finetuned["accuracy"] >= 95.0,
+            "quality_tier": self._get_quality_tier(finetuned["accuracy"])
+        }
+
+        return improvements
+
+    def _get_quality_tier(self, accuracy: float) -> str:
+        """Get quality tier based on accuracy"""
+        if accuracy >= 95:
+            return "EXCELLENT - Production Ready"
+        elif accuracy >= 85:
+            return "GOOD - Acceptable for most use cases"
+        elif accuracy >= 75:
+            return "FAIR - Needs improvement"
+        else:
+            return "POOR - Significant training needed"
+
+    def run_comparison(
+        self,
+        baseline_path: str = None,
+        finetuned_path: str = "/app/checkpoints/mcp_finetuned/final_model"
+    ) -> Dict:
+        """Run complete before/after comparison"""
+        logger.info("\n" + "=" * 60)
+        logger.info("MCP FINE-TUNING COMPARISON BENCHMARK")
+        logger.info("=" * 60)
+
+        # Get test cases
+        test_cases = self.get_test_cases()
+        logger.info(f"\nTest cases: {len(test_cases)}")
+
+        # Benchmark baseline (if provided)
+        baseline_results = None
+        if baseline_path:
+            baseline_results = self.benchmark_model(
+                model_path=baseline_path,
+                model_name="Baseline (Pre-fine-tuning)",
+                test_cases=test_cases
+            )
+
+        # Benchmark fine-tuned model
+        finetuned_results = self.benchmark_model(
+            model_path=finetuned_path,
+            model_name="Fine-tuned (MCP Tools)",
+            test_cases=test_cases
+        )
+
+        # Calculate improvements
+        improvements = None
+        if baseline_results:
+            improvements = self.calculate_improvements(baseline_results, finetuned_results)
+
+        # Create report
+        report = {
+            "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
+            "test_cases": len(test_cases),
+            "baseline": baseline_results,
+            "finetuned": finetuned_results,
+            "improvements": improvements
+        }
+
+        # Print summary
+        self._print_summary(report)
+
+        # Save report
+        report_path = self.output_dir / "finetune_comparison.json"
+        with open(report_path, "w") as f:
+            json.dump(report, f, indent=2)
+
+        logger.info(f"\n📄 Report saved to: {report_path}")
+
+        return report
+
+    def _print_summary(self, report: Dict):
+        """Print comparison summary"""
+        logger.info("\n" + "=" * 60)
+        logger.info("RESULTS SUMMARY")
+        logger.info("=" * 60)
+
+        baseline = report.get("baseline")
+        finetuned = report["finetuned"]
+        improvements = report.get("improvements")
+
+        # Fine-tuned results
+        logger.info(f"\n🎯 Fine-tuned Model:")
+        logger.info(f"  Tool Accuracy: {finetuned['accuracy']:.1f}%")
+        logger.info(f"  Parameter Accuracy: {finetuned['param_accuracy']:.1f}%")
+        logger.info(f"  Tool Detection: {finetuned['tool_found']}/{finetuned['total']}")
+        logger.info(f"  Quality Tier: {self._get_quality_tier(finetuned['accuracy'])}")
+
+        # Improvements
+        if improvements:
+            logger.info(f"\n📈 Improvements:")
+            logger.info(f"  Accuracy Gain: +{improvements['accuracy_improvement']:.1f}% ({improvements['accuracy_improvement_pct']:.1f}% improvement)")
+            logger.info(f"  Tool Detection: +{improvements['tool_detection_improvement']:.1f}%")
+            logger.info(f"  Meets 95% Target: {'✅ YES' if improvements['meets_target'] else '❌ NO'}")
+
+        # Recommendation
+        logger.info(f"\n💡 Recommendation:")
+        if finetuned['accuracy'] >= 95:
+            logger.info("  ✅ Model is production-ready for Claude Agent SDK")
+            logger.info("  ✅ Proceed with ONNX quantization and deployment")
+        elif finetuned['accuracy'] >= 85:
+            logger.info("  ✓ Model performs well for most use cases")
+            logger.info("  → Consider additional fine-tuning for critical applications")
+        else:
+            logger.info("  ⚠ Model needs improvement")
+            logger.info("  → Recommended: More training data or additional epochs")
+
+
+def main():
+    """Main entry point"""
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Compare baseline vs fine-tuned model")
+    parser.add_argument("--baseline", type=str, help="Path to baseline model (optional)")
+    parser.add_argument("--finetuned", type=str, default="/app/checkpoints/mcp_finetuned/final_model", help="Path to fine-tuned model")
+    parser.add_argument("--output-dir", type=str, default="/app/benchmarks/finetune_results", help="Output directory")
+
+    args = parser.parse_args()
+
+    # Run comparison
+    comparison = FineTuneComparison(output_dir=args.output_dir)
+    report = comparison.run_comparison(
+        baseline_path=args.baseline,
+        finetuned_path=args.finetuned
+    )
+
+    logger.info("\n✅ Comparison complete!")
+
+
+if __name__ == "__main__":
+    main()

package/agentic-llm/benchmarks/results/.gitkeep

@@ -0,0 +1,2 @@
+# Benchmark results directory
+# All training results will be stored here

package/agentic-llm/benchmarks/results/README.md

@@ -0,0 +1,59 @@
+# Benchmark Results Directory
+
+This directory contains before/after benchmarks for Phi-4 fine-tuning.
+
+## Files Generated
+
+### Baseline (Before Fine-tuning)
+- `baseline_results.json` - Baseline model performance on MCP tools
+  - Tool detection rate
+  - Tool accuracy
+  - Parameter accuracy
+  - Average inference time
+
+### Post-Training (After Fine-tuning)
+- `finetune_comparison.json` - Comparison between baseline and fine-tuned
+  - Accuracy improvements
+  - Performance metrics
+  - Tool-by-tool breakdown
+
+### Final Report
+- `final_training_report.json` - Complete training summary
+  - Training duration
+  - Success criteria validation
+  - Improvement metrics
+- `TRAINING_REPORT.md` - Human-readable report
+
+## Benchmark Methodology
+
+### Test Set
+20 validation examples covering:
+- Basic tool calls (read, write, edit)
+- Complex queries (grep, glob, bash)
+- Multi-step workflows
+- Edge cases
+
+### Metrics Tracked
+
+**Accuracy Metrics:**
+- Tool Detection Rate: % queries where tool call was detected
+- Tool Correctness: % queries with correct tool selected
+- Parameter Accuracy: % queries with correct parameters
+
+**Performance Metrics:**
+- Inference latency (ms)
+- Tokens per second
+- GPU memory usage
+
+**Quality Tiers:**
+- EXCELLENT: >95% accuracy
+- GOOD: 85-95% accuracy
+- NEEDS_IMPROVEMENT: <85% accuracy
+
+## Success Criteria
+
+✅ **Production Ready** if:
+- Tool accuracy > 95%
+- Parameter accuracy > 90%
+- Tool detection > 97%
+- Inference latency < 200ms

package/agentic-llm/benchmarks/results/agent-booster-results.json

@@ -0,0 +1,116 @@
+{
+  "metadata": {
+    "timestamp": "2025-10-07T21:37:30.120Z",
+    "runtime": "simulated",
+    "parser": "typescript-simulation",
+    "dataset": "small-test-dataset.json",
+    "samples": 12
+  },
+  "samples": [
+    {
+      "id": "test-001",
+      "description": "Add type annotations to function",
+      "latency_ms": 93,
+      "success": true,
+      "confidence": 0.9449822564157511,
+      "strategy": "fuzzy_replace"
+    },
+    {
+      "id": "test-002",
+      "description": "Add error handling to async function",
+      "latency_ms": 84,
+      "success": true,
+      "confidence": 0.9339389891105419,
+      "strategy": "fuzzy_replace"
+    },
+    {
+      "id": "test-003",
+      "description": "Convert var to const/let",
+      "latency_ms": 93,
+      "success": true,
+      "confidence": 0.9101958963966182,
+      "strategy": "fuzzy_replace"
+    },
+    {
+      "id": "test-004",
+      "description": "Add JSDoc comments",
+      "latency_ms": 83,
+      "success": true,
+      "confidence": 0.9381150179825795,
+      "strategy": "fuzzy_replace"
+    },
+    {
+      "id": "test-005",
+      "description": "Convert callback to Promise",
+      "latency_ms": 89,
+      "success": true,
+      "confidence": 0.9229781191229839,
+      "strategy": "fuzzy_replace"
+    },
+    {
+      "id": "test-006",
+      "description": "Add null checks",
+      "latency_ms": 96,
+      "success": true,
+      "confidence": 0.8724959526639541,
+      "strategy": "fuzzy_replace"
+    },
+    {
+      "id": "test-007",
+      "description": "Convert to arrow function",
+      "latency_ms": 114,
+      "success": true,
+      "confidence": 0.8741935557077134,
+      "strategy": "fuzzy_replace"
+    },
+    {
+      "id": "test-008",
+      "description": "Add input validation",
+      "latency_ms": 116,
+      "success": true,
+      "confidence": 0.9069102302845496,
+      "strategy": "fuzzy_replace"
+    },
+    {
+      "id": "test-009",
+      "description": "Convert class to TypeScript",
+      "latency_ms": 115,
+      "success": true,
+      "confidence": 0.908663172419369,
+      "strategy": "fuzzy_replace"
+    },
+    {
+      "id": "test-010",
+      "description": "Add destructuring",
+      "latency_ms": 93,
+      "success": true,
+      "confidence": 0.8741994099440088,
+      "strategy": "fuzzy_replace"
+    },
+    {
+      "id": "test-011",
+      "description": "Add try-catch wrapper",
+      "latency_ms": 92,
+      "success": true,
+      "confidence": 0.9131141055251639,
+      "strategy": "fuzzy_replace"
+    },
+    {
+      "id": "test-012",
+      "description": "Add async/await",
+      "latency_ms": 118,
+      "success": true,
+      "confidence": 0.9356269250648608,
+      "strategy": "fuzzy_replace"
+    }
+  ],
+  "aggregate": {
+    "totalLatency": 1186,
+    "avgLatency": 98.83333333333333,
+    "p50Latency": 93,
+    "p95Latency": 118,
+    "p99Latency": 118,
+    "successRate": 100,
+    "totalCost": 0
+  }
+}

package/agentic-llm/benchmarks/results/morph-baseline-results.json

@@ -0,0 +1,168 @@
+{
+  "timestamp": "2025-10-07T21:08:37.134Z",
+  "model": "morph-v3-fast",
+  "total_tests": 12,
+  "successful_tests": 12,
+  "failed_tests": 0,
+  "performance": {
+    "total_time_ms": 16238,
+    "avg_latency_ms": 352,
+    "total_api_time_ms": 4219
+  },
+  "usage": {
+    "applies_used": 12,
+    "input_tokens": 963,
+    "output_tokens": 427,
+    "remaining_budget": {
+      "applies": 488,
+      "input_tokens": 299037,
+      "output_tokens": 99573
+    }
+  },
+  "results": [
+    {
+      "test_id": "test-001",
+      "description": "Add type annotations to function",
+      "category": "typescript-conversion",
+      "success": true,
+      "output": "function add(a: number, b: number): number {\n  return a + b;\n}\n\n",
+      "latency": 466,
+      "tokens": {
+        "input": 70,
+        "output": 20
+      }
+    },
+    {
+      "test_id": "test-002",
+      "description": "Add error handling to async function",
+      "category": "error-handling",
+      "success": true,
+      "output": "async function fetchData(url) {\n  const response = await fetch(url);\n  return response.json();\n}\n\n",
+      "latency": 252,
+      "tokens": {
+        "input": 77,
+        "output": 21
+      }
+    },
+    {
+      "test_id": "test-003",
+      "description": "Convert var to const/let",
+      "category": "modernization",
+      "success": true,
+      "output": "const count = 0;\nconst name = 'John';\nfor (let i = 0; i < 10; i++) {\n  count++;\n}\n\n",
+      "latency": 285,
+      "tokens": {
+        "input": 90,
+        "output": 34
+      }
+    },
+    {
+      "test_id": "test-004",
+      "description": "Add JSDoc comments",
+      "category": "documentation",
+      "success": true,
+      "output": "/**\n * Multiply two numbers.\n * @param {number} x - The first number.\n * @param {number} y - The second number.\n * @returns {number} The product of x and y.\n */\nfunction multiply(x, y) {\n  return x * y;\n}\n\n",
+      "latency": 541,
+      "tokens": {
+        "input": 70,
+        "output": 59
+      }
+    },
+    {
+      "test_id": "test-005",
+      "description": "Convert callback to Promise",
+      "category": "async-conversion",
+      "success": true,
+      "output": "function readFile(path) {\n  return new Promise((resolve, reject) => {\n    fs.readFile(path, 'utf8', (err, data) => {\n      if (err) reject(err);\n      else resolve(data);\n    });\n  });\n}",
+      "latency": 395,
+      "tokens": {
+        "input": 98,
+        "output": 51
+      }
+    },
+    {
+      "test_id": "test-006",
+      "description": "Add null checks",
+      "category": "safety",
+      "success": true,
+      "output": "function getUserName(user) {\n  return user.profile.name;\n}\n\n",
+      "latency": 298,
+      "tokens": {
+        "input": 67,
+        "output": 14
+      }
+    },
+    {
+      "test_id": "test-007",
+      "description": "Convert to arrow function",
+      "category": "modernization",
+      "success": true,
+      "output": "const numbers = [1, 2, 3, 4, 5];\nconst doubled = numbers.map(n => n * 2);\n\n",
+      "latency": 292,
+      "tokens": {
+        "input": 90,
+        "output": 31
+      }
+    },
+    {
+      "test_id": "test-008",
+      "description": "Add input validation",
+      "category": "validation",
+      "success": true,
+      "output": "function divide(a, b) {\n  if (typeof a !== 'number' || typeof b !== 'number') {\n    throw new Error('Both arguments must be numbers');\n  }\n  return a / b;\n}",
+      "latency": 394,
+      "tokens": {
+        "input": 68,
+        "output": 45
+      }
+    },
+    {
+      "test_id": "test-009",
+      "description": "Convert class to TypeScript",
+      "category": "typescript-conversion",
+      "success": true,
+      "output": "class Calculator {\n  constructor() {\n    this.result = 0;\n  }\n  add(value) {\n    this.result += value;\n  }\n}\n\n",
+      "latency": 269,
+      "tokens": {
+        "input": 85,
+        "output": 31
+      }
+    },
+    {
+      "test_id": "test-010",
+      "description": "Add destructuring",
+      "category": "modernization",
+      "success": true,
+      "output": "function formatUser(user) {\n  const { name, email, age } = user;\n  return `${name} (${email}) - ${age} years old`;\n}\n\n",
+      "latency": 319,
+      "tokens": {
+        "input": 97,
+        "output": 35
+      }
+    },
+    {
+      "test_id": "test-011",
+      "description": "Add try-catch wrapper",
+      "category": "error-handling",
+      "success": true,
+      "output": "function parseJSON(str) {\n  try {\n    return JSON.parse(str);\n  } catch (e) {\n    return null;\n  }\n}",
+      "latency": 331,
+      "tokens": {
+        "input": 69,
+        "output": 30
+      }
+    },
+    {
+      "test_id": "test-012",
+      "description": "Add async/await",
+      "category": "async-conversion",
+      "success": true,
+      "output": "function getData() {\n  return fetch('/api/data')\n    .then(res => res.json())\n    .then(data => data);\n}\n\nasync function getData() {\n  const res = await fetch('/api/data');\n  const data = await res.json();\n  return data;\n}",
+      "latency": 377,
+      "tokens": {
+        "input": 82,
+        "output": 56
+      }
+    }
+  ]
+}