@aj-archipelago/cortex 1.4.2 → 1.4.3

package/helper-apps/cortex-autogen2/test_entity_fetcher.py

@@ -0,0 +1,305 @@
+#!/usr/bin/env python3
+"""
+Isolation test script for fetch_entity_images() function
+Tests multi-tier entity image fetcher with various scenarios
+"""
+
+import asyncio
+import json
+import os
+import sys
+import tempfile
+import shutil
+from pathlib import Path
+
+# Add project root to path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from tools.search_tools import fetch_entity_images
+
+
+class TestRunner:
+    """Runs isolation tests for fetch_entity_images"""
+
+    def __init__(self):
+        self.work_dir = tempfile.mkdtemp(prefix="entity_test_")
+        self.results = []
+        print(f"🔬 Test work directory: {self.work_dir}\n")
+
+    def cleanup(self):
+        """Clean up test directory"""
+        if os.path.exists(self.work_dir):
+            shutil.rmtree(self.work_dir)
+            print(f"\n🧹 Cleaned up test directory")
+
+    async def run_test(self, name: str, entities: list, entity_type: str, count_per_entity: int = 1, force_web_search: bool = False):
+        """Run a single test case"""
+        print("=" * 80)
+        print(f"🧪 TEST: {name}")
+        print("=" * 80)
+        print(f"   Entities: {entities}")
+        print(f"   Type: {entity_type}")
+        print(f"   Count per entity: {count_per_entity}")
+        print(f"   Force web search: {force_web_search}")
+        print()
+
+        try:
+            result_json = await fetch_entity_images(
+                entities=entities,
+                entity_type=entity_type,
+                count_per_entity=count_per_entity,
+                work_dir=self.work_dir,
+                force_web_search=force_web_search
+            )
+
+            result = json.loads(result_json)
+
+            # Print summary
+            print("📊 RESULT:")
+            print(f"   Success: {result.get('success', False)}")
+            print(f"   Total images: {result.get('stats', {}).get('total_images', 0)}")
+            print(f"   API success: {result.get('stats', {}).get('api_success', 0)}")
+            print(f"   API failed: {result.get('stats', {}).get('api_failed', 0)}")
+            print(f"   Web search used: {result.get('stats', {}).get('web_search_used', 0)}")
+
+            # Show per-entity breakdown
+            if 'results' in result:
+                print("\n   Per-entity results:")
+                for entity_data in result['results']:
+                    entity_name = entity_data.get('entity', 'unknown')
+                    img_count = len(entity_data.get('images', []))
+                    method = entity_data.get('method', 'unknown')
+                    api_used = entity_data.get('api_used', 'N/A')
+                    print(f"      {entity_name}: {img_count} images via {method} ({api_used})")
+
+            # Show sample image paths
+            if 'results' in result:
+                print("\n   Sample images:")
+                for entity_data in result['results']:
+                    entity_name = entity_data.get('entity', 'unknown')
+                    images = entity_data.get('images', [])
+                    if images:
+                        sample = images[0]
+                        filename = os.path.basename(sample.get('local_path', 'N/A'))
+                        print(f"      {entity_name}: {filename}")
+
+            if not result.get('success'):
+                print(f"\n   ⚠️  Error: {result.get('error', 'Unknown error')}")
+
+            self.results.append({
+                'name': name,
+                'success': result.get('success', False),
+                'result': result
+            })
+
+            print("\n✅ TEST PASSED\n")
+            return result
+
+        except Exception as e:
+            print(f"\n❌ TEST FAILED: {str(e)}\n")
+            import traceback
+            traceback.print_exc()
+            self.results.append({
+                'name': name,
+                'success': False,
+                'error': str(e)
+            })
+            return None
+
+    async def run_all_tests(self):
+        """Run all test scenarios"""
+        print("\n" + "=" * 80)
+        print("🚀 STARTING ISOLATION TESTS FOR fetch_entity_images()")
+        print("=" * 80)
+        print()
+
+        # TEST 1: Pokemon entities (primary use case - should use PokeAPI)
+        await self.run_test(
+            name="Pokemon - PokeAPI Integration",
+            entities=["pikachu", "charizard", "mewtwo"],
+            entity_type="pokemon",
+            count_per_entity=1
+        )
+
+        await asyncio.sleep(2)
+
+        # TEST 2: Single Pokemon with multiple images
+        await self.run_test(
+            name="Pokemon - Multiple Images per Entity",
+            entities=["gengar"],
+            entity_type="pokemon",
+            count_per_entity=3
+        )
+
+        await asyncio.sleep(2)
+
+        # TEST 3: Country entities (should use REST Countries API)
+        await self.run_test(
+            name="Countries - REST Countries API",
+            entities=["france", "japan", "brazil"],
+            entity_type="country",
+            count_per_entity=1
+        )
+
+        await asyncio.sleep(2)
+
+        # TEST 4: Invalid Pokemon name (should fallback to web search)
+        await self.run_test(
+            name="Pokemon - Invalid Name Fallback",
+            entities=["invalidpokemonxyz123"],
+            entity_type="pokemon",
+            count_per_entity=1
+        )
+
+        await asyncio.sleep(2)
+
+        # TEST 5: Unknown entity type (should fallback to web search)
+        await self.run_test(
+            name="Unknown Entity Type - Web Search Fallback",
+            entities=["tesla model 3"],
+            entity_type="car",
+            count_per_entity=2
+        )
+
+        await asyncio.sleep(2)
+
+        # TEST 6: Force web search even for known entity type
+        await self.run_test(
+            name="Pokemon - Force Web Search Override",
+            entities=["dragonite"],
+            entity_type="pokemon",
+            count_per_entity=1,
+            force_web_search=True
+        )
+
+        await asyncio.sleep(2)
+
+        # TEST 7: Mixed valid/invalid Pokemon (test fallback handling)
+        await self.run_test(
+            name="Pokemon - Mixed Valid/Invalid Entities",
+            entities=["bulbasaur", "notarealpokemon999", "squirtle"],
+            entity_type="pokemon",
+            count_per_entity=1
+        )
+
+        await asyncio.sleep(2)
+
+        # TEST 8: Empty entity list (edge case)
+        await self.run_test(
+            name="Edge Case - Empty Entity List",
+            entities=[],
+            entity_type="pokemon",
+            count_per_entity=1
+        )
+
+        await asyncio.sleep(2)
+
+        # TEST 9: Pokemon with numbers (Gen 1 powerful pokemon)
+        await self.run_test(
+            name="Pokemon - Gen 1 Powerful Pokemon",
+            entities=["mewtwo", "dragonite", "alakazam", "gengar", "zapdos"],
+            entity_type="pokemon",
+            count_per_entity=1
+        )
+
+        # Print summary
+        self.print_summary()
+
+    def print_summary(self):
+        """Print test summary"""
+        print("\n" + "=" * 80)
+        print("📊 TEST SUMMARY")
+        print("=" * 80)
+
+        passed = sum(1 for r in self.results if r.get('success'))
+        total = len(self.results)
+
+        print(f"\nTotal Tests: {total}")
+        print(f"Passed: {passed}")
+        print(f"Failed: {total - passed}")
+        print(f"Success Rate: {(passed/total*100) if total > 0 else 0:.1f}%")
+
+        print("\n📝 Individual Results:")
+        for i, result in enumerate(self.results, 1):
+            status = "✅ PASS" if result.get('success') else "❌ FAIL"
+            print(f"   {i}. {status} - {result['name']}")
+
+            if result.get('success') and 'result' in result:
+                stats = result['result'].get('stats', {})
+                total_images = stats.get('total_images', 0)
+                api_success = stats.get('api_success', 0)
+                web_search = stats.get('web_search_used', 0)
+
+                methods = []
+                if api_success > 0:
+                    methods.append(f"{api_success} via API")
+                if web_search > 0:
+                    methods.append(f"{web_search} via web search")
+
+                method_str = ", ".join(methods) if methods else "no images"
+                print(f"      → {total_images} total images ({method_str})")
+
+        print("\n" + "=" * 80)
+
+        # Check for specific success criteria
+        print("\n🎯 SUCCESS CRITERIA CHECK:")
+
+        criteria = [
+            ("Pokemon API integration works", any(
+                r.get('success') and 'Pokemon - PokeAPI' in r['name']
+                and r.get('result', {}).get('stats', {}).get('api_success', 0) > 0
+                for r in self.results
+            )),
+            ("Country API integration works", any(
+                r.get('success') and 'Countries - REST' in r['name']
+                and r.get('result', {}).get('stats', {}).get('api_success', 0) > 0
+                for r in self.results
+            )),
+            ("Web search fallback works for invalid entities", any(
+                r.get('success') and 'Invalid Name Fallback' in r['name']
+                and r.get('result', {}).get('stats', {}).get('web_search_used', 0) > 0
+                for r in self.results
+            )),
+            ("Web search fallback works for unknown types", any(
+                r.get('success') and 'Unknown Entity Type' in r['name']
+                and r.get('result', {}).get('stats', {}).get('web_search_used', 0) > 0
+                for r in self.results
+            )),
+            ("Force web search override works", any(
+                r.get('success') and 'Force Web Search' in r['name']
+                and r.get('result', {}).get('stats', {}).get('web_search_used', 0) > 0
+                for r in self.results
+            )),
+            ("Multiple images per entity works", any(
+                r.get('success') and 'Multiple Images' in r['name']
+                and r.get('result', {}).get('stats', {}).get('total_images', 0) >= 3
+                for r in self.results
+            )),
+        ]
+
+        for criterion, passed in criteria:
+            status = "✅ PASS" if passed else "❌ FAIL"
+            print(f"   {status} - {criterion}")
+
+        all_criteria_passed = all(passed for _, passed in criteria)
+
+        print("\n" + "=" * 80)
+        if all_criteria_passed:
+            print("🎉 ALL CRITERIA PASSED - Tool is ready for integration!")
+        else:
+            print("⚠️  Some criteria failed - review results above")
+        print("=" * 80)
+
+
+async def main():
+    """Main test runner"""
+    runner = TestRunner()
+
+    try:
+        await runner.run_all_tests()
+    finally:
+        runner.cleanup()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

package/helper-apps/cortex-autogen2/tests/README.md

@@ -0,0 +1,240 @@
+# Cortex AutoGen2 Automated Testing Suite
+
+Comprehensive automated testing framework for evaluating and improving the AutoGen2 system quality.
+
+## Features
+
+- ✅ **Automated Test Execution**: Run predefined test cases with zero manual intervention
+- 📊 **LLM-Based Evaluation**: Scores progress updates (0-100) and final outputs (0-100) using Cortex API
+- 📈 **Performance Metrics**: Track latency, update frequency, error rates, and more
+- 🗄️ **SQLite Storage**: All test results, scores, and metrics stored locally
+- 💡 **Improvement Suggestions**: LLM analyzes failures and suggests code improvements
+- 📉 **Trend Analysis**: Detect quality regressions over time
+- 🖥️ **CLI Interface**: Easy-to-use command-line tool
+
+## Quick Start
+
+### 1. Prerequisites
+
+Ensure you have:
+- Docker running (for cortex-autogen-function container)
+- Redis running (for progress updates)
+- Azure Queue setup
+- Environment variables configured (.env file)
+
+Required environment variables:
+```bash
+CORTEX_API_KEY=your_key_here
+CORTEX_API_BASE_URL=http://localhost:4000/v1
+REDIS_CONNECTION_STRING=redis://localhost:6379
+REDIS_CHANNEL=cortex_progress
+AZURE_STORAGE_CONNECTION_STRING=your_connection_string
+AZURE_QUEUE_NAME=cortex-tasks
+```
+
+### 2. Install Dependencies
+
+The testing suite uses the same dependencies as the main project. No additional installation needed.
+
+### 3. Run Tests
+
+```bash
+# Run all test cases
+python tests/cli/run_tests.py --all
+
+# Run specific test
+python tests/cli/run_tests.py --test tc001_pokemon_pptx
+
+# View test history
+python tests/cli/run_tests.py --history --limit 20
+
+# View score trend for a test case
+python tests/cli/run_tests.py --trend tc001_pokemon_pptx
+```
+
+## Test Cases
+
+The suite includes 3 predefined test cases:
+
+### TC001: Pokemon PPTX Presentation
+Creates a professional PowerPoint with Pokemon images, tests:
+- Image collection (10+ images)
+- Professional slide design
+- Preview image generation
+- File upload with SAS URLs
+
+### TC002: PDF Report with Images
+Generates a renewable energy PDF report, tests:
+- Web research and image collection
+- Chart/graph generation
+- PDF formatting
+- Document quality
+
+### TC003: Random CSV Generation
+Creates realistic sales data CSVs, tests:
+- Data generation
+- Statistical calculations
+- CSV formatting
+- Quick task execution
+
+## Architecture
+
+```
+tests/
+├── orchestrator.py           # Main test execution engine
+├── test_cases.yaml           # Test case definitions
+├── database/
+│   ├── schema.sql            # SQLite database schema
+│   ├── repository.py         # Data access layer
+│   └── test_results.db       # SQLite database (gitignored)
+├── collectors/
+│   ├── progress_collector.py # Redis subscriber for progress updates
+│   └── log_collector.py      # Docker log parser
+├── evaluators/
+│   ├── llm_scorer.py         # LLM-based evaluation
+│   └── prompts.py            # Evaluation prompts and rubrics
+├── metrics/
+│   └── collector.py          # Performance metrics calculation
+├── analysis/
+│   ├── improvement_suggester.py  # LLM-powered suggestions
+│   └── trend_analyzer.py     # Trend and regression detection
+└── cli/
+    └── run_tests.py          # CLI interface
+```
+
+## How It Works
+
+1. **Test Submission**: Test orchestrator submits task to Azure Queue
+2. **Data Collection**:
+   - Progress collector subscribes to Redis for real-time updates
+   - Log collector streams Docker container logs
+3. **Execution Monitoring**: Wait for task completion or timeout
+4. **Data Storage**: Store progress updates, logs, files in SQLite
+5. **Metrics Calculation**: Calculate latency, frequency, error counts
+6. **LLM Evaluation**:
+   - Score progress updates (frequency, clarity, accuracy)
+   - Score final output (completeness, quality, correctness)
+7. **Analysis**: Generate improvement suggestions and track trends
+
+## Evaluation Criteria
+
+### Progress Updates (0-100)
+- **Frequency** (25 pts): Updates every 2-5 seconds ideal
+- **Clarity** (25 pts): Emojis, concise, informative
+- **Accuracy** (25 pts): Progress % matches work done
+- **Coverage** (25 pts): All important steps communicated
+
+### Final Output (0-100)
+- **Completeness** (25 pts): All deliverables present
+- **Quality** (25 pts): Professional, polished, no placeholders
+- **Correctness** (25 pts): Accurate data, no hallucinations
+- **Presentation** (25 pts): SAS URLs, previews, clear results
+
+## Database Schema
+
+Test results are stored in `tests/database/test_results.db`:
+
+- **test_runs**: Test execution records
+- **progress_updates**: Real-time progress data
+- **logs**: Docker log entries
+- **files_created**: Generated files with SAS URLs
+- **evaluations**: LLM scores and reasoning
+- **metrics**: Performance metrics
+- **suggestions**: Improvement recommendations
+
+## Example Output
+
+```
+🧪 Running Test: Pokemon PowerPoint Presentation with Images
+   ID: tc001_pokemon_pptx
+   Timeout: 300s
+
+📝 Test run created: ID=1, Request=test_tc001_pokemon_pptx_a3f9b12e
+✅ Task submitted to queue
+📡 Starting data collection...
+   Progress: 10% - 📋 Planning task execution...
+   Progress: 25% - 🌐 Collecting Pokemon images...
+   Progress: 50% - 💻 Creating PowerPoint presentation...
+   Progress: 75% - 📸 Generating slide previews...
+   Progress: 100% - ✅ Task completed successfully!
+✅ Data collection complete
+   Progress updates: 12
+   Log entries: 45
+
+📊 Calculating metrics...
+   Time to completion: 142.3s
+   Progress updates: 12
+   Files created: 15
+   Errors: 0
+
+🤖 Running LLM evaluation...
+   Progress Score: 88/100
+   Output Score: 92/100
+
+✨ Evaluation complete:
+   Progress Score: 88/100
+   Output Score: 92/100
+   Overall Score: 90/100
+
+✅ Test Complete: Pokemon PowerPoint Presentation with Images
+```
+
+## Extending the Suite
+
+### Add New Test Cases
+
+Edit `tests/test_cases.yaml`:
+
+```yaml
+test_cases:
+  - id: tc004_my_new_test
+    name: "My New Test"
+    task: "Test task description..."
+    timeout_seconds: 300
+    expected_deliverables:
+      - type: pdf
+        pattern: "*.pdf"
+        min_count: 1
+    min_progress_updates: 5
+    quality_criteria:
+      - "Criterion 1"
+      - "Criterion 2"
+```
+
+### Customize Evaluation
+
+Modify prompts in `tests/evaluators/prompts.py` to change scoring criteria.
+
+### Add New Metrics
+
+Extend `tests/metrics/collector.py` with additional metrics calculation logic.
+
+## Troubleshooting
+
+### No progress updates collected
+- Check Redis is running: `redis-cli ping`
+- Verify REDIS_CONNECTION_STRING in .env
+- Check Docker container is running: `docker ps`
+
+### Database errors
+- Delete and recreate: `rm tests/database/test_results.db`
+- Schema will auto-recreate on next run
+
+### LLM evaluation fails
+- Verify CORTEX_API_KEY is set
+- Check CORTEX_API_BASE_URL is accessible
+- Review logs for API errors
+
+## Future Enhancements
+
+- [ ] Web dashboard for viewing results
+- [ ] CI/CD integration (GitHub Actions)
+- [ ] Parallel test execution
+- [ ] Screenshot comparison for visual regression
+- [ ] Custom test case generator
+- [ ] Export reports (PDF, HTML)
+- [ ] Slack/email notifications
+
+## License
+
+Part of the Cortex AutoGen2 project.