agentic-flow 1.9.0 → 1.9.2

package/agentic-llm/cloudrun/cloud_runner.py

@@ -1,390 +0,0 @@
-#!/usr/bin/env python3
-"""
-Cloud Run GPU Training Orchestrator
-Manages fine-tuning pipeline with health reporting and email notifications
-"""
-
-import os
-import sys
-import json
-import time
-import logging
-from pathlib import Path
-from datetime import datetime
-
-# Add parent directory to path
-sys.path.append(str(Path(__file__).parent.parent))
-
-from training.finetune_mcp import MCPFineTuner, MCPFineTuningConfig
-from validation.mcp_validator import MCPValidator
-from benchmarks.finetune_comparison import MCPBenchmark
-
-import resend
-
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-)
-logger = logging.getLogger(__name__)
-
-
-class CloudRunOrchestrator:
-    """Orchestrate training, validation, and benchmarking on Cloud Run"""
-
-    def __init__(self):
-        self.start_time = datetime.now()
-        self.status_file = Path("/app/results/training_status.json")
-        self.status_file.parent.mkdir(parents=True, exist_ok=True)
-
-        # Email configuration
-        self.resend_api_key = os.environ.get("RESEND_API_KEY")
-        self.notification_email = os.environ.get("NOTIFICATION_EMAIL")
-
-        if self.resend_api_key:
-            resend.api_key = self.resend_api_key
-            logger.info(f"Email notifications enabled for: {self.notification_email}")
-        else:
-            logger.warning("RESEND_API_KEY not set - email notifications disabled")
-
-    def update_status(self, phase: str, status: str, details: dict = None):
-        """Update training status for monitoring"""
-        status_data = {
-            "phase": phase,
-            "status": status,
-            "timestamp": datetime.now().isoformat(),
-            "elapsed_seconds": (datetime.now() - self.start_time).total_seconds(),
-            "details": details or {}
-        }
-
-        with open(self.status_file, 'w') as f:
-            json.dump(status_data, f, indent=2)
-
-        logger.info(f"Status update: {phase} - {status}")
-
-    def run_baseline_benchmarks(self):
-        """Run baseline benchmarks on original Phi-4"""
-        logger.info("=" * 60)
-        logger.info("PHASE 1: Baseline Benchmarking")
-        logger.info("=" * 60)
-
-        self.update_status("baseline_benchmarks", "running")
-
-        try:
-            benchmark = MCPBenchmark()
-            baseline_results = benchmark.run_baseline_benchmark()
-
-            # Save baseline results
-            baseline_file = Path("/app/benchmarks/results/baseline_results.json")
-            baseline_file.parent.mkdir(parents=True, exist_ok=True)
-
-            with open(baseline_file, 'w') as f:
-                json.dump(baseline_results, f, indent=2)
-
-            self.update_status("baseline_benchmarks", "completed", baseline_results)
-            logger.info(f"Baseline results saved: {baseline_file}")
-
-            return baseline_results
-
-        except Exception as e:
-            logger.error(f"Baseline benchmark failed: {e}")
-            self.update_status("baseline_benchmarks", "failed", {"error": str(e)})
-            raise
-
-    def run_finetuning(self):
-        """Execute fine-tuning pipeline"""
-        logger.info("=" * 60)
-        logger.info("PHASE 2: Fine-Tuning")
-        logger.info("=" * 60)
-
-        self.update_status("finetuning", "running")
-
-        try:
-            config = MCPFineTuningConfig(
-                output_dir="/app/checkpoints/mcp_finetuned",
-                model_cache_dir="/app/models"
-            )
-
-            finetuner = MCPFineTuner(config)
-            metrics = finetuner.train()
-
-            self.update_status("finetuning", "completed", metrics)
-            logger.info("Fine-tuning completed successfully")
-
-            return metrics
-
-        except Exception as e:
-            logger.error(f"Fine-tuning failed: {e}")
-            self.update_status("finetuning", "failed", {"error": str(e)})
-            raise
-
-    def run_validation(self):
-        """Validate fine-tuned model"""
-        logger.info("=" * 60)
-        logger.info("PHASE 3: Validation")
-        logger.info("=" * 60)
-
-        self.update_status("validation", "running")
-
-        try:
-            validator = MCPValidator(
-                model_path="/app/checkpoints/mcp_finetuned/final_model",
-                output_dir="/app/validation/results"
-            )
-
-            results = validator.run_validation()
-
-            self.update_status("validation", "completed", results)
-            logger.info(f"Validation results: {results}")
-
-            return results
-
-        except Exception as e:
-            logger.error(f"Validation failed: {e}")
-            self.update_status("validation", "failed", {"error": str(e)})
-            raise
-
-    def run_post_training_benchmarks(self):
-        """Run benchmarks on fine-tuned model"""
-        logger.info("=" * 60)
-        logger.info("PHASE 4: Post-Training Benchmarking")
-        logger.info("=" * 60)
-
-        self.update_status("post_training_benchmarks", "running")
-
-        try:
-            benchmark = MCPBenchmark()
-            comparison_results = benchmark.run_comparison(
-                finetuned_model_path="/app/checkpoints/mcp_finetuned/final_model"
-            )
-
-            # Save comparison results
-            comparison_file = Path("/app/benchmarks/results/finetune_comparison.json")
-
-            with open(comparison_file, 'w') as f:
-                json.dump(comparison_results, f, indent=2)
-
-            self.update_status("post_training_benchmarks", "completed", comparison_results)
-            logger.info(f"Comparison results saved: {comparison_file}")
-
-            return comparison_results
-
-        except Exception as e:
-            logger.error(f"Post-training benchmark failed: {e}")
-            self.update_status("post_training_benchmarks", "failed", {"error": str(e)})
-            raise
-
-    def send_completion_email(self, report: dict):
-        """Send email notification when training completes"""
-        if not self.resend_api_key or not self.notification_email:
-            logger.warning("Email notification skipped - credentials not configured")
-            return
-
-        try:
-            success = report['success_criteria']['passed']
-            status_emoji = "✅" if success else "⚠️"
-
-            # Create HTML email
-            html_content = f"""
-            <html>
-            <body style="font-family: Arial, sans-serif; max-width: 800px; margin: 0 auto;">
-                <h1 style="color: #333;">{status_emoji} Phi-4 Fine-tuning Complete</h1>
-
-                <div style="background: #f5f5f5; padding: 20px; border-radius: 8px; margin: 20px 0;">
-                    <h2 style="margin-top: 0;">Training Summary</h2>
-                    <ul>
-                        <li><strong>Duration:</strong> {report['training_summary']['total_duration_hours']:.2f} hours</li>
-                        <li><strong>Status:</strong> {'SUCCESS' if success else 'NEEDS REVIEW'}</li>
-                        <li><strong>Started:</strong> {report['training_summary']['start_time']}</li>
-                        <li><strong>Completed:</strong> {report['training_summary']['end_time']}</li>
-                    </ul>
-                </div>
-
-                <div style="background: {'#d4edda' if success else '#fff3cd'}; padding: 20px; border-radius: 8px; margin: 20px 0;">
-                    <h2 style="margin-top: 0;">Performance Metrics</h2>
-                    <ul>
-                        <li><strong>Tool Accuracy:</strong> {report['validation_results'].get('accuracy', 0):.1f}% (Target: 95%)</li>
-                        <li><strong>Parameter Accuracy:</strong> {report['validation_results'].get('param_accuracy', 0):.1f}%</li>
-                        <li><strong>Quality Tier:</strong> {report['validation_results'].get('quality_tier', 'Unknown')}</li>
-                    </ul>
-                </div>
-
-                <div style="background: #e7f3ff; padding: 20px; border-radius: 8px; margin: 20px 0;">
-                    <h2 style="margin-top: 0;">Baseline vs Fine-tuned</h2>
-                    <p>Improvement metrics available in detailed report.</p>
-                </div>
-
-                <div style="margin-top: 30px; padding: 15px; background: #f9f9f9; border-left: 4px solid #007bff;">
-                    <p style="margin: 0;"><strong>Next Steps:</strong></p>
-                    <ul style="margin-bottom: 0;">
-                        <li>Review detailed results in Cloud Run logs</li>
-                        <li>Download model from: <code>/app/checkpoints/mcp_finetuned/final_model/</code></li>
-                        <li>{'Upload to Hugging Face: python3 cloudrun/upload_to_huggingface.py' if success else 'Review training parameters and re-run'}</li>
-                        <li>{'Deploy to production' if success else 'Adjust hyperparameters'}</li>
-                    </ul>
-                </div>
-
-                <hr style="margin: 30px 0; border: none; border-top: 1px solid #ddd;">
-
-                <p style="color: #666; font-size: 12px;">
-                    This is an automated notification from your Phi-4 fine-tuning pipeline on Google Cloud Run.
-                </p>
-            </body>
-            </html>
-            """
-
-            params = {
-                "from": "Agentic Flow <onboarding@resend.dev>",
-                "to": [self.notification_email],
-                "subject": f"{status_emoji} Phi-4 Fine-tuning Complete - {'Success' if success else 'Needs Review'}",
-                "html": html_content,
-                "tags": [
-                    {"name": "project", "value": "phi4-finetuning"},
-                    {"name": "status", "value": "success" if success else "review"}
-                ]
-            }
-
-            email_response = resend.Emails.send(params)
-            logger.info(f"Email notification sent successfully: {email_response}")
-
-        except Exception as e:
-            logger.error(f"Failed to send email notification: {e}")
-
-    def generate_final_report(self, baseline, validation, comparison):
-        """Generate comprehensive final report"""
-        logger.info("=" * 60)
-        logger.info("PHASE 5: Final Report Generation")
-        logger.info("=" * 60)
-
-        report = {
-            "training_summary": {
-                "start_time": self.start_time.isoformat(),
-                "end_time": datetime.now().isoformat(),
-                "total_duration_hours": (datetime.now() - self.start_time).total_seconds() / 3600
-            },
-            "baseline_performance": baseline,
-            "validation_results": validation,
-            "improvement_metrics": comparison,
-            "success_criteria": {
-                "tool_accuracy_target": 95.0,
-                "tool_accuracy_achieved": validation.get("accuracy", 0),
-                "passed": validation.get("accuracy", 0) >= 95.0
-            }
-        }
-
-        # Save final report
-        report_file = Path("/app/benchmarks/results/final_training_report.json")
-
-        with open(report_file, 'w') as f:
-            json.dump(report, f, indent=2)
-
-        # Generate markdown summary
-        self._generate_markdown_report(report)
-
-        # Send email notification
-        self.send_completion_email(report)
-
-        logger.info(f"Final report saved: {report_file}")
-        return report
-
-    def _generate_markdown_report(self, report):
-        """Generate human-readable markdown report"""
-        markdown = f"""# Phi-4 Fine-Tuning Results
-
-## Training Summary
-- **Start Time**: {report['training_summary']['start_time']}
-- **End Time**: {report['training_summary']['end_time']}
-- **Duration**: {report['training_summary']['total_duration_hours']:.2f} hours
-
-## Baseline Performance
-```json
-{json.dumps(report['baseline_performance'], indent=2)}
-```
-
-## Validation Results
-- **Tool Accuracy**: {report['validation_results'].get('accuracy', 0):.1f}%
-- **Parameter Accuracy**: {report['validation_results'].get('param_accuracy', 0):.1f}%
-- **Quality Tier**: {report['validation_results'].get('quality_tier', 'Unknown')}
-
-## Success Criteria
-- **Target**: {report['success_criteria']['tool_accuracy_target']}% tool accuracy
-- **Achieved**: {report['success_criteria']['tool_accuracy_achieved']}%
-- **Status**: {'✅ PASSED' if report['success_criteria']['passed'] else '❌ FAILED'}
-
-## Improvement Metrics
-```json
-{json.dumps(report['improvement_metrics'], indent=2)}
-```
-
----
-Generated on Cloud Run GPU instance
-"""
-
-        markdown_file = Path("/app/benchmarks/results/TRAINING_REPORT.md")
-        with open(markdown_file, 'w') as f:
-            f.write(markdown)
-
-        logger.info(f"Markdown report saved: {markdown_file}")
-
-    def run_complete_pipeline(self):
-        """Execute complete training and evaluation pipeline"""
-        logger.info("=" * 80)
-        logger.info("CLOUD RUN GPU TRAINING PIPELINE - PHI-4 FINE-TUNING")
-        logger.info("=" * 80)
-
-        try:
-            # Phase 1: Baseline benchmarks
-            baseline = self.run_baseline_benchmarks()
-
-            # Phase 2: Fine-tuning
-            training_metrics = self.run_finetuning()
-
-            # Phase 3: Validation
-            validation = self.run_validation()
-
-            # Phase 4: Post-training benchmarks
-            comparison = self.run_post_training_benchmarks()
-
-            # Phase 5: Final report
-            final_report = self.generate_final_report(baseline, validation, comparison)
-
-            self.update_status("pipeline", "completed", final_report)
-
-            logger.info("=" * 80)
-            logger.info("PIPELINE COMPLETED SUCCESSFULLY")
-            logger.info(f"Total duration: {final_report['training_summary']['total_duration_hours']:.2f} hours")
-            logger.info(f"Tool accuracy: {validation.get('accuracy', 0):.1f}%")
-            logger.info("=" * 80)
-
-            return final_report
-
-        except Exception as e:
-            logger.error(f"Pipeline failed: {e}")
-            self.update_status("pipeline", "failed", {"error": str(e)})
-            raise
-
-
-def main():
-    """Main entry point for Cloud Run"""
-    logger.info("Starting Cloud Run training orchestrator...")
-
-    orchestrator = CloudRunOrchestrator()
-
-    try:
-        final_report = orchestrator.run_complete_pipeline()
-
-        # Keep container alive for result retrieval
-        logger.info("Training complete. Container will remain available for result retrieval.")
-        logger.info("Results available at: /app/benchmarks/results/")
-
-        # Sleep to keep container alive (Cloud Run will handle shutdown)
-        while True:
-            time.sleep(60)
-            logger.info("Container still running. Results available.")
-
-    except Exception as e:
-        logger.error(f"Fatal error: {e}")
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

package/agentic-llm/cloudrun/deploy.sh

@@ -1,83 +0,0 @@
-#!/bin/bash
-# Deploy Phi-4 fine-tuning to Google Cloud Run with GPU
-
-set -e
-
-# Configuration
-PROJECT_ID=${GOOGLE_CLOUD_PROJECT:-"agentic-llm"}
-REGION=${REGION:-"us-central1"}
-SERVICE_NAME="phi4-finetuning-gpu"
-IMAGE_NAME="gcr.io/${PROJECT_ID}/${SERVICE_NAME}"
-GPU_TYPE="nvidia-l4"  # L4 GPU (recommended for Cloud Run)
-MEMORY="16Gi"
-CPU="4"
-TIMEOUT="3600s"  # 1 hour (can be extended)
-
-echo "========================================="
-echo "Cloud Run GPU Deployment - Phi-4"
-echo "========================================="
-echo "Project: ${PROJECT_ID}"
-echo "Region: ${REGION}"
-echo "Service: ${SERVICE_NAME}"
-echo "GPU: ${GPU_TYPE}"
-echo "========================================="
-
-# Check prerequisites
-command -v gcloud >/dev/null 2>&1 || { echo "gcloud CLI not found. Install from https://cloud.google.com/sdk"; exit 1; }
-
-# Set project
-echo "Setting project..."
-gcloud config set project ${PROJECT_ID}
-
-# Enable required APIs
-echo "Enabling required APIs..."
-gcloud services enable \
-    cloudbuild.googleapis.com \
-    run.googleapis.com \
-    containerregistry.googleapis.com \
-    compute.googleapis.com
-
-# Build container
-echo "Building Docker image..."
-gcloud builds submit \
-    --tag ${IMAGE_NAME} \
-    --timeout=30m \
-    --machine-type=e2-highcpu-8 \
-    -f cloudrun/Dockerfile.gpu \
-    .
-
-# Deploy to Cloud Run with GPU
-echo "Deploying to Cloud Run with GPU..."
-gcloud run deploy ${SERVICE_NAME} \
-    --image ${IMAGE_NAME} \
-    --platform managed \
-    --region ${REGION} \
-    --gpu ${GPU_TYPE} \
-    --gpu-count 1 \
-    --memory ${MEMORY} \
-    --cpu ${CPU} \
-    --timeout ${TIMEOUT} \
-    --min-instances 0 \
-    --max-instances 1 \
-    --allow-unauthenticated \
-    --set-env-vars "PYTHONUNBUFFERED=1,CUDA_VISIBLE_DEVICES=0" \
-    --set-secrets "HF_TOKEN=huggingface-token:latest,RESEND_API_KEY=resend-api-key:latest,NOTIFICATION_EMAIL=notification-email:latest" \
-    --no-cpu-throttling
-
-# Get service URL
-SERVICE_URL=$(gcloud run services describe ${SERVICE_NAME} --region ${REGION} --format 'value(status.url)')
-
-echo "========================================="
-echo "Deployment Complete!"
-echo "========================================="
-echo "Service URL: ${SERVICE_URL}"
-echo ""
-echo "Monitor training:"
-echo "  curl ${SERVICE_URL}/status"
-echo ""
-echo "View logs:"
-echo "  gcloud run logs read ${SERVICE_NAME} --region ${REGION} --limit 50"
-echo ""
-echo "Get results:"
-echo "  curl ${SERVICE_URL}/results"
-echo "========================================="

package/agentic-llm/cloudrun/entrypoint.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-# Entrypoint for Cloud Run
-# Health server starts immediately and handles training in background thread
-
-echo "🚀 Starting Cloud Run GPU Training Service"
-echo "📡 Health server will respond immediately on port ${PORT:-8080}"
-echo "⏳ Training will start automatically after 30 seconds"
-
-# Run health server (which triggers training internally)
-exec python3 /app/cloudrun/health_server.py

package/agentic-llm/cloudrun/health_server.py

@@ -1,114 +0,0 @@
-#!/usr/bin/env python3
-"""
-Health check server for Cloud Run
-Provides status endpoint for monitoring
-Starts training in background thread after server is healthy
-"""
-
-import json
-import os
-import subprocess
-import threading
-import time
-from pathlib import Path
-from flask import Flask, jsonify
-from datetime import datetime
-
-app = Flask(__name__)
-
-STATUS_FILE = Path("/app/results/training_status.json")
-TRAINING_STARTED = False
-
-
-@app.route("/")
-def index():
-    """Root endpoint"""
-    return jsonify({
-        "service": "phi4-finetuning",
-        "status": "running",
-        "timestamp": datetime.now().isoformat()
-    })
-
-
-@app.route("/health")
-def health():
-    """Health check endpoint for Cloud Run"""
-    return jsonify({"status": "healthy"}), 200
-
-
-@app.route("/status")
-def status():
-    """Training status endpoint"""
-    if STATUS_FILE.exists():
-        with open(STATUS_FILE, 'r') as f:
-            status_data = json.load(f)
-        return jsonify(status_data), 200
-    else:
-        return jsonify({
-            "status": "initializing",
-            "message": "Training not yet started"
-        }), 200
-
-
-@app.route("/results")
-def results():
-    """Get training results if available"""
-    results_dir = Path("/app/benchmarks/results")
-
-    if not results_dir.exists():
-        return jsonify({"error": "Results not available yet"}), 404
-
-    available_files = list(results_dir.glob("*.json"))
-
-    return jsonify({
-        "results_available": len(available_files) > 0,
-        "files": [f.name for f in available_files],
-        "path": str(results_dir)
-    }), 200
-
-
-def start_training_async():
-    """Start training in background thread after 30 second delay"""
-    global TRAINING_STARTED
-
-    # Wait 30 seconds to ensure health checks pass
-    print("⏳ Waiting 30 seconds before starting training...")
-    time.sleep(30)
-
-    print("🚀 Starting training in background...")
-    TRAINING_STARTED = True
-
-    try:
-        # Run training script
-        process = subprocess.Popen(
-            ["python3", "/app/cloudrun/cloud_runner.py"],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            text=True
-        )
-
-        # Stream output to log file
-        with open("/app/logs/training.log", "w") as log_file:
-            for line in process.stdout:
-                print(line, end="")
-                log_file.write(line)
-                log_file.flush()
-
-        process.wait()
-        print(f"✅ Training completed with exit code: {process.returncode}")
-
-    except Exception as e:
-        print(f"❌ Training failed: {e}")
-
-
-if __name__ == "__main__":
-    port = int(os.environ.get("PORT", 8080))
-
-    # Start training in background thread
-    training_thread = threading.Thread(target=start_training_async, daemon=True)
-    training_thread.start()
-    print(f"✅ Health server starting on port {port}")
-    print(f"📋 Training will start in 30 seconds...")
-
-    # Run Flask server in main thread
-    app.run(host="0.0.0.0", port=port, debug=False)

package/agentic-llm/cloudrun/setup-secrets.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-# Setup Google Cloud Secrets for Cloud Run deployment
-
-set -e
-
-echo "Setting up Google Cloud Secrets..."
-
-# Load environment variables
-if [ -f "/workspaces/agentic-flow/.env" ]; then
-    export $(cat /workspaces/agentic-flow/.env | grep -E "^(RESEND_API_KEY|NOTIFICATION_EMAIL|HUGGINGFACE_API_KEY)=" | xargs)
-fi
-
-# Create secrets
-echo "Creating resend-api-key secret..."
-echo -n "${RESEND_API_KEY}" | gcloud secrets create resend-api-key --data-file=- --replication-policy=automatic 2>&1 || \
-    echo -n "${RESEND_API_KEY}" | gcloud secrets versions add resend-api-key --data-file=-
-
-echo "Creating notification-email secret..."
-echo -n "${NOTIFICATION_EMAIL}" | gcloud secrets create notification-email --data-file=- --replication-policy=automatic 2>&1 || \
-    echo -n "${NOTIFICATION_EMAIL}" | gcloud secrets versions add notification-email --data-file=-
-
-echo "Creating huggingface-token secret..."
-echo -n "${HUGGINGFACE_API_KEY}" | gcloud secrets create huggingface-token --data-file=- --replication-policy=automatic 2>&1 || \
-    echo -n "${HUGGINGFACE_API_KEY}" | gcloud secrets versions add huggingface-token --data-file=-
-
-echo "✅ Secrets created successfully!"