@aj-archipelago/cortex 1.3.62 → 1.3.63

package/helper-apps/cortex-autogen2/pyproject.toml

@@ -0,0 +1,36 @@
+[tool.poetry]
+name = "cortex-autogen2"
+version = "0.1.0"
+description = "Multi-agent coding assistant using AutoGen"
+authors = ["Your Name <your.email@example.com>"]
+readme = "README.md"
+packages = [{include = "cortex_autogen2", from = "src"}]
+
+[tool.poetry.dependencies]
+python = ">=3.11,<3.13"
+autogen-agentchat = {extras = ["openai"], version = "^0.6.4"}
+python-dotenv = "^1.0.1"
+requests = "^2.32.3"
+redis = "^5.0.4"
+azure-storage-queue = "^12.10.1"
+azure-storage-blob = "^12.19.0"
+sqlalchemy = "^2.0.30"
+Pillow = "^11.0.0"
+pymysql = "^1.1.1"
+playwright = "^1.54.0"
+markitdown = "^0.1.2"
+docker = "^7.1.0"
+autogen-ext = {extras = ["docker"], version = "^0.6.4"}
+openai = "^1.97.1"
+tiktoken = "^0.9.0"
+aiofiles = "^24.1.0"
+pandas = "^2.3.1"
+matplotlib = "^3.10.3"
+aiohttp = "^3.12.14"
+
+[tool.poetry.group.dev.dependencies]
+ipykernel = "^6.29.5"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api" 

package/helper-apps/cortex-autogen2/requirements.txt

@@ -0,0 +1,20 @@
+azure-functions>=1.20.0
+autogen-agentchat[openai]>=0.6.4
+python-dotenv>=1.0.1
+requests>=2.32.3
+redis>=5.0.4
+azure-storage-queue>=12.10.1
+azure-storage-blob>=12.19.0
+sqlalchemy>=2.0.30
+Pillow>=11.0.0
+pymysql>=1.1.1
+playwright>=1.54.0
+markitdown>=0.1.2
+docker>=7.1.0
+autogen-ext[docker]>=0.6.4
+openai>=1.97.1
+tiktoken>=0.9.0
+aiofiles>=24.1.0
+aiohttp>=3.12.14 
+pandas
+matplotlib 

package/helper-apps/cortex-autogen2/send_task.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""
+🚀 Cortex AutoGen Task Sender
+
+Send a task to the Azure Storage Queue for processing by the Cortex AutoGen worker.
+
+CRITICAL: Clean worker state prevents conflicts
+----------------------------------------------
+1) Kill existing workers:
+   pkill -f "python -m src.cortex_autogen2.main" || true
+   pkill -f "python main.py" || true
+
+2) Start fresh worker (non-continuous) in background:
+   CONTINUOUS_MODE=false python -m src.cortex_autogen2.main &
+   # or: CONTINUOUS_MODE=false python main.py &
+
+3) Send a task:
+   python send_task.py "Create a simple PDF about cats and upload it"
+
+Why this order? Multiple workers can cause duplicate processing, queue conflicts, and noisy progress updates.
+
+Usage
+-----
+python send_task.py "<your task text>" [--queue <name>] [--connection <conn_str>]
+
+Environment
+-----------
+- AZURE_STORAGE_CONNECTION_STRING (required if --connection not provided)
+- AZURE_QUEUE_NAME (default queue if --queue not provided)
+- .env is loaded automatically
+
+Message format
+--------------
+- Base64-encoded JSON payload with a `content` field:
+  {"request_id": "<uuid>", "message_id": "<uuid>", "content": "<task text>"}
+
+Notes
+-----
+- Tasks persist in the queue until a worker processes them.
+- Progress is published to Redis (see README for details).
+"""
+
+import json
+import uuid
+import argparse
+import base64
+from azure.storage.queue import QueueClient
+from dotenv import load_dotenv
+import os
+
+def main():
+    """Sends a simple task to the Azure Queue."""
+    load_dotenv()
+    
+    parser = argparse.ArgumentParser(description="Send a task to the AutoGen agent processor.")
+    parser.add_argument(
+        "task_prompt",
+        type=str,
+        nargs='?',
+        default="list the files in the current directory",
+        help="The prompt for the task to be executed."
+    )
+    parser.add_argument(
+        "--queue",
+        dest="queue_name",
+        type=str,
+        default=os.getenv("AZURE_QUEUE_NAME", "autogen-test-message-queue"),
+        help="Azure Storage Queue name (overrides AZURE_QUEUE_NAME)."
+    )
+    parser.add_argument(
+        "--connection",
+        dest="connection_string",
+        type=str,
+        default=os.getenv("AZURE_STORAGE_CONNECTION_STRING"),
+        help="Azure Storage connection string (overrides AZURE_STORAGE_CONNECTION_STRING)."
+    )
+    args = parser.parse_args()
+
+    connection_string = args.connection_string
+    queue_name = args.queue_name
+    print(f"Using queue: {queue_name}")
+    
+    if not connection_string:
+        print("Error: AZURE_STORAGE_CONNECTION_STRING is not set and --connection was not provided.")
+        return
+        
+    task = {
+        "request_id": str(uuid.uuid4()),
+        "message_id": str(uuid.uuid4()),
+        "content": args.task_prompt
+    }
+    
+    message = json.dumps(task)
+    
+    # Encode message as Base64 to match Azure Functions MessageEncoding setting
+    encoded_message = base64.b64encode(message.encode('utf-8')).decode('utf-8')
+    
+    # Use synchronous client for instant execution
+    queue_client = QueueClient.from_connection_string(connection_string, queue_name)
+    queue_client.send_message(encoded_message)
+    print(f"Task sent to queue '{queue_name}': {task}")
+    print(f"Message encoded as Base64: {encoded_message[:50]}...")
+
+if __name__ == "__main__":
+    main() 

package/helper-apps/cortex-autogen2/services/__init__.py

@@ -0,0 +1 @@
+ 

package/helper-apps/cortex-autogen2/services/azure_queue.py

@@ -0,0 +1,85 @@
+import asyncio
+from azure.storage.queue.aio import QueueServiceClient, QueueClient
+from azure.core.exceptions import ResourceExistsError, AzureError
+import os
+import logging
+
+logger = logging.getLogger(__name__)
+
+class AzureQueueService:
+    """
+    A service for interacting with Azure Queue Storage.
+    """
+
+    def __init__(self, connection_string: str, queue_name: str):
+        self.connection_string = connection_string
+        self.queue_name = queue_name
+        self.queue_client = QueueClient.from_connection_string(
+            conn_str=self.connection_string, queue_name=self.queue_name
+        )
+
+    async def initialize(self):
+        """
+        Initializes the queue, creating it if it doesn't exist.
+        """
+        try:
+            await self.queue_client.create_queue()
+        except ResourceExistsError:
+            pass
+        except Exception as e:
+            logger.error(f"💥 Failed to create or connect to queue '{self.queue_name}': {e}")
+            raise
+
+    async def get_task(self) -> dict | None:
+        """
+        Receives a single message from the queue.
+        """
+        try:
+            messages = self.queue_client.receive_messages(
+                messages_per_page=1, 
+                visibility_timeout=1800,
+                timeout=30
+            )
+            
+            async for message in messages:
+                logger.info(f"📨 Azure Queue: Received message with ID: {message.id}")
+                return {
+                    "id": message.id,
+                    "content": message.content,
+                    "pop_receipt": message.pop_receipt,
+                }
+            return None
+
+        except AzureError as e:
+            logger.error(f"❌ Azure Queue: An Azure-specific error occurred: {e}")
+            return None
+        except Exception as e:
+            logger.error(f"❌ Azure Queue: Unexpected error receiving message: {e}", exc_info=True)
+            return None
+
+    async def delete_task(self, message_id: str, pop_receipt: str):
+        """
+        Deletes a message from the queue after it has been processed.
+        """
+        try:
+            await self.queue_client.delete_message(message_id, pop_receipt)
+        except Exception as e:
+            logger.error(f"💥 Failed to delete message {message_id}: {e}")
+            raise
+
+    async def close(self):
+        """
+        Closes the QueueClient.
+        """
+        await self.queue_client.close()
+
+async def get_queue_service() -> AzureQueueService:
+    """
+    Factory function to create and initialize an AzureQueueService instance.
+    """
+    connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
+    queue_name = os.getenv("AZURE_QUEUE_NAME")
+    
+    queue_service = AzureQueueService(connection_string, queue_name)
+    await queue_service.initialize()
+    return queue_service 

package/helper-apps/cortex-autogen2/services/redis_publisher.py

@@ -0,0 +1,153 @@
+import redis
+import json
+import logging
+from typing import Dict, Any, Optional
+
+import os
+
+logger = logging.getLogger(__name__)
+
+# Global Redis client - persistent connection like the working version
+redis_client = None
+
+def connect_redis() -> bool:
+    """Check and ensure Redis connection is active - matches working version pattern"""
+    global redis_client
+    
+    redis_conn_string = os.getenv("REDIS_CONNECTION_STRING")
+
+    # Initialize client if not exists
+    if redis_client is None:
+        try:
+            redis_client = redis.from_url(redis_conn_string)
+        except Exception as e:
+            logger.warning(f"Failed to create Redis client: {e}")
+            return False
+    
+    # Test connection
+    try:
+        redis_client.ping()
+        return True
+    except redis.ConnectionError as e:
+        logger.warning(f"Redis connection error: {e}")
+        try:
+            # Try to reconnect
+            redis_client = redis.from_url(redis_conn_string)
+            redis_client.ping()
+            return True
+        except Exception as reconnect_error:
+            logger.error(f"Error reconnecting to Redis: {reconnect_error}")
+            return False
+    except Exception as e:
+        logger.warning(f"Redis ping failed: {e}")
+        # Handle the case where client is closed
+        if "Client must be connected" in str(e) or "closed" in str(e).lower():
+            logger.info("Redis client was closed, attempting to create new connection...")
+            try:
+                redis_client = redis.from_url(redis_conn_string)
+                redis_client.ping()
+                return True
+            except Exception as reconnect_error:
+                logger.error(f"Error creating new Redis connection: {reconnect_error}")
+                return False
+        return False
+
+def publish_request_progress(data: Dict[str, Any]) -> bool:
+    """Publish progress data to Redis channel - matches working version pattern"""
+    if connect_redis():
+        try:
+            message = json.dumps(data)
+            result = redis_client.publish(os.getenv("REDIS_CHANNEL"), message)
+            logger.info(f"Published progress update for request {data.get('requestId')}: progress={data.get('progress')}, subscribers={result}")
+            return True
+        except Exception as e:
+            logger.error(f"Error publishing message to Redis: {e}")
+            return False
+    else:
+        logger.error(f"Redis not connected, failed to publish progress update for request {data.get('requestId')}")
+        return False
+
+class RedisPublisher:
+    """Wrapper class for compatibility with existing code"""
+    
+    def __init__(self):
+        self.connected = False
+        
+    async def connect(self):
+        """Initialize Redis connection"""
+        self.connected = connect_redis()
+        if self.connected:
+            logger.info("Connected to Redis successfully")
+        else:
+            logger.warning("Failed to connect to Redis")
+            logger.warning("Redis progress publishing will be disabled")
+    
+    def publish_request_progress(self, data: Dict[str, Any]) -> bool:
+        """Publish progress data to Redis channel"""
+        return publish_request_progress(data)
+    
+    async def publish_progress(self, request_id: str, progress: float, info: str = "", data: str = None) -> bool:
+        """Publish progress update for a specific request - async version"""
+        message_data = {
+            "requestId": request_id,
+            "progress": progress,
+            "info": info
+        }
+        
+        # Add data field for final results
+        if data is not None:
+            message_data["data"] = data
+            
+        return self.publish_request_progress(message_data)
+    
+    def store_final_result(self, request_id: str, result_data: Dict[str, Any], expiry_seconds: int = 3600) -> bool:
+        """Store final result in Redis key for retrieval"""
+        if connect_redis():
+            try:
+                # Store in multiple keys for compatibility
+                keys_to_store = [
+                    f"result:{request_id}",
+                    f"final:{request_id}",
+                    f"progress:{request_id}"  # Also store in progress key for consistency
+                ]
+                
+                message = json.dumps(result_data)
+                
+                for key in keys_to_store:
+                    redis_client.set(key, message, ex=expiry_seconds)
+                
+                logger.info(f"Stored final result for request {request_id} in {len(keys_to_store)} Redis keys")
+                return True
+                
+            except Exception as e:
+                logger.error(f"Error storing final result: {e}")
+                return False
+        else:
+            logger.debug(f"Redis not connected, skipping final result storage for request {request_id}")
+            return False
+    
+    async def close(self):
+        """Close Redis connection gracefully"""
+        global redis_client
+        if redis_client:
+            try:
+                # Don't actually close the connection in non-continuous mode
+                # Just mark it as disconnected so it can be recreated if needed
+                logger.info("Redis connection marked for cleanup")
+                # Only close if we're in continuous mode or shutting down completely
+                # redis_client.close()  # Comment out to prevent premature closure
+                # redis_client = None
+            except Exception as e:
+                logger.warning(f"Error during Redis connection cleanup: {e}")
+        self.connected = False
+
+# Global instance
+_redis_publisher: Optional[RedisPublisher] = None
+
+async def get_redis_publisher() -> RedisPublisher:
+    """Get or create Redis publisher instance"""
+    global _redis_publisher
+    if _redis_publisher is None:
+        _redis_publisher = RedisPublisher()
+        await _redis_publisher.connect()
+    return _redis_publisher