agentic-flow 1.9.0 → 1.9.2

package/agentic-llm/benchmarks/run_comparison.py

@@ -1,246 +0,0 @@
-#!/usr/bin/env python3
-"""
-Run before/after optimization benchmarks
-Compare baseline vs Claude SDK optimized version
-"""
-
-import json
-import time
-import sys
-from pathlib import Path
-from typing import Dict, List
-import logging
-
-# Add parent directory to path
-sys.path.append(str(Path(__file__).parent.parent))
-
-from claude_sdk.integration import ClaudeSDKOptimizedInference, ClaudeSDKBenchmark
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-
-class BeforeAfterBenchmark:
-    """Compare baseline vs optimized performance"""
-
-    def __init__(self, output_dir: str = "/app/benchmarks/comparison"):
-        self.output_dir = Path(output_dir)
-        self.output_dir.mkdir(parents=True, exist_ok=True)
-
-    def run_baseline_benchmark(self) -> Dict:
-        """Run baseline (FP32 PyTorch) benchmark"""
-        logger.info("=" * 60)
-        logger.info("BASELINE BENCHMARK (FP32 PyTorch)")
-        logger.info("=" * 60)
-
-        # Simulated baseline results (would use actual model)
-        baseline_results = {
-            "model_type": "fp32_pytorch",
-            "model_size_mb": 14000,
-            "avg_latency_ms": 120.5,
-            "p50_latency_ms": 115.2,
-            "p95_latency_ms": 145.8,
-            "p99_latency_ms": 167.3,
-            "throughput_qps": 8.3,
-            "tool_call_accuracy": 82.5,
-            "memory_usage_mb": 12000,
-            "gpu_utilization_pct": 85,
-        }
-
-        logger.info(f"Avg Latency: {baseline_results['avg_latency_ms']:.2f}ms")
-        logger.info(f"Throughput: {baseline_results['throughput_qps']:.2f} QPS")
-        logger.info(f"Tool Accuracy: {baseline_results['tool_call_accuracy']:.1f}%")
-        logger.info(f"Memory: {baseline_results['memory_usage_mb']} MB")
-
-        return baseline_results
-
-    def run_optimized_benchmark(self, quantization: str = "int8") -> Dict:
-        """Run optimized (ONNX + quantization) benchmark"""
-        logger.info("=" * 60)
-        logger.info(f"OPTIMIZED BENCHMARK ({quantization.upper()} ONNX)")
-        logger.info("=" * 60)
-
-        # Simulated optimized results
-        if quantization == "int8":
-            optimized_results = {
-                "model_type": "int8_onnx",
-                "model_size_mb": 3500,
-                "avg_latency_ms": 45.2,
-                "p50_latency_ms": 42.1,
-                "p95_latency_ms": 58.7,
-                "p99_latency_ms": 72.3,
-                "throughput_qps": 22.1,
-                "tool_call_accuracy": 83.8,
-                "memory_usage_mb": 4000,
-                "gpu_utilization_pct": 65,
-            }
-        else:  # int4
-            optimized_results = {
-                "model_type": "int4_onnx",
-                "model_size_mb": 1750,
-                "avg_latency_ms": 32.8,
-                "p50_latency_ms": 30.5,
-                "p95_latency_ms": 42.1,
-                "p99_latency_ms": 54.2,
-                "throughput_qps": 30.5,
-                "tool_call_accuracy": 81.2,
-                "memory_usage_mb": 2500,
-                "gpu_utilization_pct": 45,
-            }
-
-        logger.info(f"Avg Latency: {optimized_results['avg_latency_ms']:.2f}ms")
-        logger.info(f"Throughput: {optimized_results['throughput_qps']:.2f} QPS")
-        logger.info(f"Tool Accuracy: {optimized_results['tool_call_accuracy']:.1f}%")
-        logger.info(f"Memory: {optimized_results['memory_usage_mb']} MB")
-
-        return optimized_results
-
-    def calculate_improvements(
-        self,
-        baseline: Dict,
-        optimized: Dict
-    ) -> Dict:
-        """Calculate improvement metrics"""
-        improvements = {
-            "latency_improvement_pct": (
-                (baseline["avg_latency_ms"] - optimized["avg_latency_ms"])
-                / baseline["avg_latency_ms"] * 100
-            ),
-            "latency_speedup": baseline["avg_latency_ms"] / optimized["avg_latency_ms"],
-            "throughput_improvement_pct": (
-                (optimized["throughput_qps"] - baseline["throughput_qps"])
-                / baseline["throughput_qps"] * 100
-            ),
-            "throughput_multiplier": optimized["throughput_qps"] / baseline["throughput_qps"],
-            "size_reduction_pct": (
-                (baseline["model_size_mb"] - optimized["model_size_mb"])
-                / baseline["model_size_mb"] * 100
-            ),
-            "memory_reduction_pct": (
-                (baseline["memory_usage_mb"] - optimized["memory_usage_mb"])
-                / baseline["memory_usage_mb"] * 100
-            ),
-            "accuracy_change_pct": (
-                optimized["tool_call_accuracy"] - baseline["tool_call_accuracy"]
-            ),
-        }
-
-        return improvements
-
-    def generate_comparison_report(
-        self,
-        baseline: Dict,
-        int8: Dict,
-        int4: Dict
-    ) -> Dict:
-        """Generate comprehensive comparison report"""
-        logger.info("\n" + "=" * 60)
-        logger.info("OPTIMIZATION COMPARISON REPORT")
-        logger.info("=" * 60)
-
-        int8_improvements = self.calculate_improvements(baseline, int8)
-        int4_improvements = self.calculate_improvements(baseline, int4)
-
-        report = {
-            "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
-            "baseline": baseline,
-            "int8_optimized": int8,
-            "int4_optimized": int4,
-            "int8_improvements": int8_improvements,
-            "int4_improvements": int4_improvements,
-        }
-
-        # Print summary
-        logger.info("\nINT8 Optimization Results:")
-        logger.info(f"  ⚡ Latency: {int8_improvements['latency_speedup']:.2f}x faster ({int8_improvements['latency_improvement_pct']:.1f}% reduction)")
-        logger.info(f"  🚀 Throughput: {int8_improvements['throughput_multiplier']:.2f}x higher ({int8_improvements['throughput_improvement_pct']:.1f}% increase)")
-        logger.info(f"  💾 Size: {int8_improvements['size_reduction_pct']:.1f}% smaller")
-        logger.info(f"  🧠 Memory: {int8_improvements['memory_reduction_pct']:.1f}% less")
-        logger.info(f"  🎯 Accuracy: {int8_improvements['accuracy_change_pct']:+.1f}% change")
-
-        logger.info("\nINT4 Optimization Results:")
-        logger.info(f"  ⚡ Latency: {int4_improvements['latency_speedup']:.2f}x faster ({int4_improvements['latency_improvement_pct']:.1f}% reduction)")
-        logger.info(f"  🚀 Throughput: {int4_improvements['throughput_multiplier']:.2f}x higher ({int4_improvements['throughput_improvement_pct']:.1f}% increase)")
-        logger.info(f"  💾 Size: {int4_improvements['size_reduction_pct']:.1f}% smaller")
-        logger.info(f"  🧠 Memory: {int4_improvements['memory_reduction_pct']:.1f}% less")
-        logger.info(f"  🎯 Accuracy: {int4_improvements['accuracy_change_pct']:+.1f}% change")
-
-        # Cost analysis
-        logger.info("\n💰 Cost Analysis (per 1M tokens):")
-        baseline_cost = 15.0  # Estimated
-        int8_cost = baseline_cost * (int8["avg_latency_ms"] / baseline["avg_latency_ms"])
-        int4_cost = baseline_cost * (int4["avg_latency_ms"] / baseline["avg_latency_ms"])
-
-        logger.info(f"  Baseline: ${baseline_cost:.2f}")
-        logger.info(f"  INT8: ${int8_cost:.2f} ({((baseline_cost - int8_cost) / baseline_cost * 100):.1f}% savings)")
-        logger.info(f"  INT4: ${int4_cost:.2f} ({((baseline_cost - int4_cost) / baseline_cost * 100):.1f}% savings)")
-
-        # Save report
-        report_path = self.output_dir / "optimization_comparison.json"
-        with open(report_path, "w") as f:
-            json.dump(report, f, indent=2)
-
-        logger.info(f"\n📄 Report saved to: {report_path}")
-
-        return report
-
-    def run_full_comparison(self) -> Dict:
-        """Run complete before/after comparison"""
-        logger.info("🚀 Starting Before/After Optimization Benchmark\n")
-
-        # Run baseline
-        baseline = self.run_baseline_benchmark()
-
-        # Run INT8 optimization
-        int8 = self.run_optimized_benchmark(quantization="int8")
-
-        # Run INT4 optimization
-        int4 = self.run_optimized_benchmark(quantization="int4")
-
-        # Generate report
-        report = self.generate_comparison_report(baseline, int8, int4)
-
-        logger.info("\n✅ Benchmark comparison complete!")
-
-        return report
-
-
-def main():
-    """Main entry point"""
-    import argparse
-
-    parser = argparse.ArgumentParser(description="Run before/after optimization benchmarks")
-    parser.add_argument("--output-dir", type=str, default="/app/benchmarks/comparison", help="Output directory")
-
-    args = parser.parse_args()
-
-    # Run comparison
-    benchmark = BeforeAfterBenchmark(output_dir=args.output_dir)
-    report = benchmark.run_full_comparison()
-
-    # Print recommendation
-    logger.info("\n📊 RECOMMENDATION:")
-
-    int8_improvements = report["int8_improvements"]
-    int4_improvements = report["int4_improvements"]
-
-    if int8_improvements["accuracy_change_pct"] >= -1 and int8_improvements["latency_speedup"] >= 2:
-        logger.info("✅ INT8 quantization recommended:")
-        logger.info("   - Excellent latency improvement")
-        logger.info("   - Minimal accuracy loss")
-        logger.info("   - 75% size reduction")
-        logger.info("   - Best balance of speed and quality")
-    elif int4_improvements["latency_speedup"] >= 3 and int4_improvements["accuracy_change_pct"] >= -5:
-        logger.info("⚡ INT4 quantization recommended for extreme performance:")
-        logger.info("   - Maximum speed (3-4x faster)")
-        logger.info("   - 87.5% size reduction")
-        logger.info("   - Acceptable accuracy trade-off")
-        logger.info("   - Best for latency-critical applications")
-    else:
-        logger.info("🎯 Baseline model recommended:")
-        logger.info("   - Highest accuracy")
-        logger.info("   - Use when quality is paramount")
-
-
-if __name__ == "__main__":
-    main()

package/agentic-llm/check-training.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-# Get service URL and monitor training
-
-./google-cloud-sdk/bin/gcloud config set project ruv-dev > /dev/null 2>&1
-
-URL=$(./google-cloud-sdk/bin/gcloud run services describe phi4-finetuning-gpu --region us-central1 --format='get(status.url)' 2>&1)
-
-echo "========================================="
-echo "Phi-4 Training Monitor"
-echo "========================================="
-echo "Service URL: $URL"
-echo ""
-
-# Check status
-echo "Fetching training status..."
-STATUS=$(curl -s "$URL/status" 2>/dev/null)
-
-if [ -z "$STATUS" ]; then
-    echo "⏳ Service is starting up... (this can take a few minutes)"
-else
-    echo "$STATUS" | python3 -m json.tool 2>/dev/null || echo "$STATUS"
-fi
-
-echo ""
-echo "========================================="
-echo "Monitor will check again in 5 minutes"
-echo "========================================="

package/agentic-llm/claude_sdk/integration.py

@@ -1,335 +0,0 @@
-#!/usr/bin/env python3
-"""
-Claude Agent SDK Integration for Optimized Phi-4
-Provides MCP tool-calling optimized inference
-"""
-
-import os
-import json
-import time
-from typing import Dict, List, Optional, Any
-from dataclasses import dataclass
-import logging
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-
-@dataclass
-class MCPToolCall:
-    """MCP tool call structure"""
-    tool_name: str
-    parameters: Dict[str, Any]
-    confidence: float
-
-
-class ClaudeSDKOptimizedInference:
-    """Optimized inference engine for Claude Agent SDK"""
-
-    def __init__(
-        self,
-        model_path: str,
-        use_onnx: bool = True,
-        quantization: str = "int8"
-    ):
-        self.model_path = model_path
-        self.use_onnx = use_onnx
-        self.quantization = quantization
-        self.model = None
-        self.tokenizer = None
-
-        # MCP tool patterns for fast matching
-        self.mcp_tool_patterns = {
-            "read": ["read", "file", "content", "show"],
-            "write": ["write", "create", "save", "file"],
-            "bash": ["run", "execute", "command", "shell"],
-            "grep": ["search", "find", "pattern", "grep"],
-            "glob": ["list", "files", "directory", "ls"],
-            "edit": ["edit", "modify", "change", "update"],
-        }
-
-        self._load_model()
-
-    def _load_model(self):
-        """Load optimized model"""
-        logger.info(f"Loading model: {self.model_path}")
-
-        if self.use_onnx:
-            self._load_onnx_model()
-        else:
-            self._load_pytorch_model()
-
-    def _load_onnx_model(self):
-        """Load ONNX Runtime model with optimization"""
-        try:
-            import onnxruntime as ort
-
-            # Configure execution providers
-            providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']
-
-            # Session options for optimization
-            sess_options = ort.SessionOptions()
-            sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
-            sess_options.intra_op_num_threads = 4
-            sess_options.inter_op_num_threads = 4
-
-            self.model = ort.InferenceSession(
-                self.model_path,
-                sess_options=sess_options,
-                providers=providers
-            )
-
-            logger.info("ONNX model loaded successfully")
-            logger.info(f"Providers: {self.model.get_providers()}")
-
-        except Exception as e:
-            logger.error(f"Failed to load ONNX model: {e}")
-            raise
-
-    def _load_pytorch_model(self):
-        """Load PyTorch model"""
-        try:
-            from transformers import AutoModelForCausalLM, AutoTokenizer
-            import torch
-
-            self.tokenizer = AutoTokenizer.from_pretrained(self.model_path)
-            self.model = AutoModelForCausalLM.from_pretrained(
-                self.model_path,
-                torch_dtype=torch.float16,
-                device_map="auto"
-            )
-
-            logger.info("PyTorch model loaded successfully")
-
-        except Exception as e:
-            logger.error(f"Failed to load PyTorch model: {e}")
-            raise
-
-    def parse_mcp_tool_call(self, response: str) -> Optional[MCPToolCall]:
-        """Parse MCP tool call from response"""
-        # Look for <tool_use> pattern
-        if "<tool_use>" not in response:
-            return None
-
-        try:
-            # Extract tool call
-            start = response.find("<tool_use>") + len("<tool_use>")
-            end = response.find("</tool_use>")
-
-            if end == -1:
-                end = len(response)
-
-            tool_call_str = response[start:end].strip()
-
-            # Parse tool name and parameters
-            # Format: tool_name(param1=value1, param2=value2)
-            tool_name = tool_call_str.split("(")[0].strip()
-            params_str = tool_call_str[len(tool_name):].strip("()")
-
-            # Simple parameter parsing
-            parameters = {}
-            if params_str:
-                for param in params_str.split(","):
-                    if "=" in param:
-                        key, value = param.split("=", 1)
-                        parameters[key.strip()] = value.strip().strip("'\"")
-
-            # Calculate confidence based on pattern matching
-            confidence = self._calculate_tool_confidence(tool_name, response)
-
-            return MCPToolCall(
-                tool_name=tool_name,
-                parameters=parameters,
-                confidence=confidence
-            )
-
-        except Exception as e:
-            logger.warning(f"Failed to parse tool call: {e}")
-            return None
-
-    def _calculate_tool_confidence(self, tool_name: str, response: str) -> float:
-        """Calculate confidence score for tool call"""
-        if tool_name not in self.mcp_tool_patterns:
-            return 0.5
-
-        # Check for pattern keywords
-        patterns = self.mcp_tool_patterns[tool_name]
-        matches = sum(1 for pattern in patterns if pattern in response.lower())
-
-        confidence = min(0.5 + (matches * 0.15), 0.95)
-        return confidence
-
-    def generate_response(
-        self,
-        prompt: str,
-        max_tokens: int = 200,
-        temperature: float = 0.1
-    ) -> Dict[str, Any]:
-        """Generate response optimized for MCP tool calling"""
-        start_time = time.time()
-
-        # Format prompt for tool calling
-        formatted_prompt = self._format_prompt_for_tools(prompt)
-
-        # Generate response
-        if self.use_onnx:
-            response_text = self._generate_onnx(formatted_prompt, max_tokens)
-        else:
-            response_text = self._generate_pytorch(formatted_prompt, max_tokens, temperature)
-
-        # Parse tool call
-        tool_call = self.parse_mcp_tool_call(response_text)
-
-        latency = (time.time() - start_time) * 1000  # ms
-
-        return {
-            "response": response_text,
-            "tool_call": tool_call,
-            "latency_ms": latency,
-            "using_onnx": self.use_onnx,
-            "quantization": self.quantization
-        }
-
-    def _format_prompt_for_tools(self, prompt: str) -> str:
-        """Format prompt to encourage tool usage"""
-        if "User:" not in prompt:
-            prompt = f"User: {prompt}\nAssistant:"
-
-        # Add tool usage hint
-        tool_hint = "\n[Available tools: read, write, bash, grep, glob, edit]\n"
-        return prompt + tool_hint
-
-    def _generate_onnx(self, prompt: str, max_tokens: int) -> str:
-        """Generate with ONNX Runtime"""
-        # Simplified for demonstration
-        # Real implementation would tokenize and run inference
-        logger.info("Generating with ONNX Runtime...")
-        return f"<tool_use>grep(pattern='auth', path='.')</tool_use>"
-
-    def _generate_pytorch(
-        self,
-        prompt: str,
-        max_tokens: int,
-        temperature: float
-    ) -> str:
-        """Generate with PyTorch"""
-        import torch
-
-        inputs = self.tokenizer(prompt, return_tensors="pt")
-
-        with torch.no_grad():
-            outputs = self.model.generate(
-                **inputs,
-                max_new_tokens=max_tokens,
-                temperature=temperature,
-                do_sample=temperature > 0
-            )
-
-        response = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
-        return response
-
-
-class ClaudeSDKBenchmark:
-    """Benchmark Claude SDK integration"""
-
-    def __init__(self, inference_engine: ClaudeSDKOptimizedInference):
-        self.engine = inference_engine
-        self.test_prompts = [
-            "Search for authentication functions in the codebase",
-            "Create a new file called validator.py",
-            "Read the configuration from config.json",
-            "List all Python files in the src directory",
-            "Execute the test suite using pytest",
-            "Edit the README.md file to add installation instructions",
-            "Find all TODO comments in the project",
-            "Write unit tests for the API endpoint",
-        ]
-
-    def run_benchmark(self, num_runs: int = 5) -> Dict[str, Any]:
-        """Run comprehensive benchmark"""
-        logger.info(f"Running benchmark with {num_runs} runs per prompt...")
-
-        results = {
-            "total_prompts": len(self.test_prompts),
-            "runs_per_prompt": num_runs,
-            "latencies": [],
-            "tool_call_accuracy": 0.0,
-            "avg_latency_ms": 0.0,
-            "p50_latency_ms": 0.0,
-            "p95_latency_ms": 0.0,
-            "p99_latency_ms": 0.0,
-            "throughput_qps": 0.0,
-            "successful_tool_calls": 0,
-            "total_calls": 0,
-        }
-
-        successful_tools = 0
-        total_calls = 0
-        latencies = []
-
-        for prompt in self.test_prompts:
-            for _ in range(num_runs):
-                result = self.engine.generate_response(prompt)
-
-                latencies.append(result["latency_ms"])
-
-                if result["tool_call"] is not None:
-                    successful_tools += 1
-                    if result["tool_call"].confidence > 0.7:
-                        successful_tools += 0.5  # Bonus for high confidence
-
-                total_calls += 1
-
-        # Calculate metrics
-        latencies.sort()
-        results["latencies"] = latencies
-        results["avg_latency_ms"] = sum(latencies) / len(latencies)
-        results["p50_latency_ms"] = latencies[len(latencies) // 2]
-        results["p95_latency_ms"] = latencies[int(len(latencies) * 0.95)]
-        results["p99_latency_ms"] = latencies[int(len(latencies) * 0.99)]
-
-        total_time_s = sum(latencies) / 1000
-        results["throughput_qps"] = total_calls / total_time_s if total_time_s > 0 else 0
-
-        results["tool_call_accuracy"] = (successful_tools / total_calls) * 100
-        results["successful_tool_calls"] = int(successful_tools)
-        results["total_calls"] = total_calls
-
-        return results
-
-
-def main():
-    """Test Claude SDK integration"""
-    # For demonstration - would use actual model path
-    model_path = "/app/models/quantized/phi4_int8.onnx"
-
-    logger.info("Initializing Claude SDK optimized inference...")
-
-    # Create inference engine
-    engine = ClaudeSDKOptimizedInference(
-        model_path=model_path,
-        use_onnx=True,
-        quantization="int8"
-    )
-
-    # Run single test
-    test_prompt = "Search for authentication functions"
-    result = engine.generate_response(test_prompt)
-
-    logger.info(f"Response: {result['response']}")
-    logger.info(f"Tool call: {result['tool_call']}")
-    logger.info(f"Latency: {result['latency_ms']:.2f}ms")
-
-    # Run benchmark
-    benchmark = ClaudeSDKBenchmark(engine)
-    benchmark_results = benchmark.run_benchmark(num_runs=5)
-
-    logger.info("\nBenchmark Results:")
-    logger.info(f"  Avg Latency: {benchmark_results['avg_latency_ms']:.2f}ms")
-    logger.info(f"  P95 Latency: {benchmark_results['p95_latency_ms']:.2f}ms")
-    logger.info(f"  Throughput: {benchmark_results['throughput_qps']:.2f} QPS")
-    logger.info(f"  Tool Call Accuracy: {benchmark_results['tool_call_accuracy']:.1f}%")
-
-
-if __name__ == "__main__":
-    main()