@aj-archipelago/cortex 1.2.1 → 1.3.0

package/config.js

@@ -6,6 +6,7 @@ import { fileURLToPath, pathToFileURL } from 'url';
 import GcpAuthTokenHelper from './lib/gcpAuthTokenHelper.js';
 import logger from './lib/logger.js';
 import PathwayManager from './lib/pathwayManager.js';
+import { readdir } from 'fs/promises';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -221,7 +222,7 @@ var config = convict({
                 "type": "REPLICATE-API",
                 "url": "https://api.replicate.com/v1/models/black-forest-labs/flux-1.1-pro/predictions",
                 "headers": {
-                    "Prefer": "wait",
+                    "Prefer": "wait=60",
                     "Authorization": "Token {{REPLICATE_API_KEY}}",
                     "Content-Type": "application/json"
                 },
@@ -230,7 +231,7 @@ var config = convict({
                 "type": "REPLICATE-API",
                 "url": "https://api.replicate.com/v1/models/black-forest-labs/flux-schnell/predictions",
                 "headers": {
-                    "Prefer": "wait",
+                    "Prefer": "wait=10",
                     "Authorization": "Token {{REPLICATE_API_KEY}}",
                     "Content-Type": "application/json"
                 },
@@ -411,25 +412,51 @@ const createDynamicPathwayManager = async (config, basePathway) => {
 const buildPathways = async (config) => {
     const { pathwaysPath, corePathwaysPath, basePathwayPath } = config.getProperties();
 
-    const pathwaysURL = pathToFileURL(pathwaysPath).toString();
-    const corePathwaysURL = pathToFileURL(corePathwaysPath).toString();
     const basePathwayURL = pathToFileURL(basePathwayPath).toString();
-
+    
     // Load cortex base pathway 
     const basePathway = await import(basePathwayURL).then(module => module.default);
 
-    // Load core pathways, default from the Cortex package
-    logger.info(`Loading core pathways from ${corePathwaysPath}`)
-    let loadedPathways = await import(`${corePathwaysURL}/index.js`).then(module => module);
+    // Helper function to recursively load pathway files
+    const loadPathwaysFromDir = async (dirPath) => {
+        const pathways = {};
+        try {
+            const files = await readdir(dirPath, { withFileTypes: true });
+            
+            for (const file of files) {
+                const fullPath = path.join(dirPath, file.name);
+                if (file.isDirectory()) {
+                    // Skip the shared directory
+                    if (file.name === 'shared') continue;
+                    
+                    // Recursively load pathways from other subdirectories
+                    const subPathways = await loadPathwaysFromDir(fullPath);
+                    Object.assign(pathways, subPathways);
+                } else if (file.name.endsWith('.js')) {
+                    // Load individual pathway file
+                    const pathwayURL = pathToFileURL(fullPath).toString();
+                    const pathway = await import(pathwayURL).then(module => module.default || module);
+                    const pathwayName = path.basename(file.name, '.js');
+                    pathways[pathwayName] = pathway;
+                }
+            }
+        } catch (error) {
+            logger.error(`Error loading pathways from ${dirPath}: ${error.message}`);
+        }
+        return pathways;
+    };
+
+    // Load core pathways
+    logger.info(`Loading core pathways from ${corePathwaysPath}`);
+    let loadedPathways = await loadPathwaysFromDir(corePathwaysPath);
 
     // Load custom pathways and override core pathways if same
     if (pathwaysPath && fs.existsSync(pathwaysPath)) {
-        logger.info(`Loading custom pathways from ${pathwaysPath}`)
-        const customPathways = await import(`${pathwaysURL}/index.js`).then(module => module);
+        logger.info(`Loading custom pathways from ${pathwaysPath}`);
+        const customPathways = await loadPathwaysFromDir(pathwaysPath);
         loadedPathways = { ...loadedPathways, ...customPathways };
     }
 
-
     const { DYNAMIC_PATHWAYS_CONFIG_FILE, DYNAMIC_PATHWAYS_CONFIG_JSON } = process.env;
 
     let dynamicPathwayConfig;

package/helper-apps/cortex-autogen/OAI_CONFIG_LIST

@@ -1,5 +1,6 @@
 [
     {
-        "model": "claude-3.5-sonnet"
+        "model": "gpt-4o",
+        "price": [0,0]
     }
 ]

package/helper-apps/cortex-autogen/agents.py

@@ -0,0 +1,387 @@
+from autogen import AssistantAgent, UserProxyAgent, config_list_from_json
+from utils import publish_request_progress, zip_and_upload_tmp_folder
+from prompts import *
+from data_operations import store_in_mongo
+from search import search_index, index_message
+from config import *
+import os
+import logging
+import json
+import tempfile
+import time
+from datetime import datetime, timezone
+import autogen.coding
+from data_operations import check_for_human_input
+from agents_extra import process_helper_results
+from config import prompts
+import queue
+import threading
+
+shared_queues = {}
+def background_human_input_check(request_id):
+    while True:
+        human_input = check_for_human_input(request_id)
+        if human_input:
+            shared_queues[request_id].put(human_input)
+            if human_input in ["TERMINATE", "PAUSE"]:
+                break
+        time.sleep(1)
+
+
+def find_code_message(all_messages):
+    if not all_messages or len(all_messages) < 2:
+        return ""
+    
+    failed = False
+    code_message = ""
+    
+    for i in range(len(all_messages)):
+        current_message = all_messages[i].get('message') or all_messages[i].get('content', '')
+        failed = failed or "(execution failed)\n" in current_message
+
+        if not failed and "(execution failed)\n" in current_message:
+            failed = True
+
+        if failed:
+            if "exitcode: 0 (execution succeeded)" in current_message:
+                #grap 4 messages including the current one
+                messages = all_messages[i-4:i+1]
+                code_message = "\n".join([(msg['message'] or msg['content']) for msg in messages])
+                return code_message
+    return ""
+
+
+def is_termination_msg(m):
+    content = m.get("content", "").strip()
+    if not content or content.rstrip().endswith("TERMINATE") or content.startswith("exitcode: 0 (execution succeeded)"):
+        return True
+    return False
+
+
+#use this via chat() function
+def chat_with_agents(**kwargs):
+    prompt = kwargs.pop("prompt", None)
+    message = kwargs.pop("message", None)
+
+    if kwargs.pop("add_python_coder_prompt", True):
+        prompt += prompts.get("PYTHON_CODER_SYSTEM_MESSAGE")
+
+    if kwargs.pop("add_never_hallucinate_prompt", True):
+        prompt += prompts.get("NEVER_HALLUCINATE_SYSTEM_MESSAGE")
+
+    if kwargs.pop("add_current_datetime_prompt", True):
+        current_datetime = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
+        CURRENT_DATETIME_PROMPT = f"""
+        You know that current date and time is {current_datetime}. 
+        """
+        prompt += CURRENT_DATETIME_PROMPT
+
+    if not message:
+        logging.warning("No message provided! Skipping chat!")
+        return
+    if not prompt:
+        logging.warning("No prompt provided!")
+
+    original_request_message = kwargs.pop("original_request_message", None)
+    original_request_message_data = kwargs.pop("original_request_message_data", None)
+
+    llm_config = kwargs.pop("llm_config", None)
+    request_id = kwargs.pop("request_id", None)
+    chat_publish_progress = kwargs.pop("chat_publish_progress", None)
+    
+    request_reply = kwargs.pop("request_reply", None)
+    silent = kwargs.pop("silent", True)
+
+    recipient = kwargs.pop("recipient", None)
+
+    return_type = kwargs.pop("return_type", "last")
+
+    all_messages = kwargs.pop("all_messages", None)
+    if all_messages is None:
+        logging.warning("No all_messages list provided!")
+        all_messages = []
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        code_executor = autogen.coding.LocalCommandLineCodeExecutor(work_dir=temp_dir,timeout=300)
+
+        assistant = AssistantAgent("assistant", llm_config=llm_config, system_message=prompt, is_termination_msg=is_termination_msg)
+        user_proxy = UserProxyAgent("user_proxy", human_input_mode="NEVER", max_consecutive_auto_reply=20,
+                                    code_execution_config={"executor": code_executor},
+                                    is_termination_msg=is_termination_msg)
+        
+        def create_send_function(agent):
+            nonlocal request_reply, silent, request_id, chat_publish_progress, all_messages, return_type, message, recipient, user_proxy, assistant
+            original_send = agent.send
+            def send(message, recipient, request_reply=None, silent=True):
+                return logged_send(agent, original_send, message, recipient, request_reply, silent, request_id, chat_publish_progress, all_messages)
+            return send
+
+        assistant.send = create_send_function(assistant)
+        user_proxy.send = create_send_function(user_proxy)
+
+        chat_result = user_proxy.initiate_chat(
+            assistant, 
+            message=message, 
+        )
+
+
+
+    code_msg = find_code_message(all_messages)
+    if code_msg:
+        try:
+            corrector = AssistantAgent("code_corrector", llm_config=llm_config, system_message=prompts.get("CODE_CORRECTOR_PROMPTER_SYSTEM_MESSAGE"))
+            corrector_result = corrector.generate_reply(messages=[{"content": code_msg, "role":"user"}])
+
+            logging.info(f"Code corrector result: {corrector_result}")
+
+            index_message({
+                "requestId": request_id,
+                "content":corrector_result, #code_msg,
+                "task": original_request_message,
+                "contextId": original_request_message_data.get("contextId"),
+                "requestId": request_id,
+            })
+        except Exception as e:
+            logging.error(f"Error extracting code corrector result: {e}")
+
+    if return_type == "chat_history":
+        return chat_result.chat_history
+    if return_type == "chat_result":
+        return chat_result
+    if return_type == "summary":
+        return chat_result.summary
+    if return_type == "last":
+        return chat_result.chat_history[-1]["content"] or chat_result.chat_history[-2]["content"]
+    if return_type == "all_as_str":
+        return "\n".join([msg['content'] for msg in chat_result.chat_history])
+
+    return chat_result 
+
+
+
+def logged_send(sender, original_send, message, recipient, request_reply=None, silent=True, request_id=None, chat_publish_progress=None, all_messages=None):
+    if not message:
+        logging.info("Empty message, skipping!")
+        return
+    if not request_id:
+        logging.warning("No request_id provided!")
+
+    all_messages.append({
+        "sender": sender.name,
+        "message": message
+    })
+
+    if chat_publish_progress:
+        chat_publish_progress({
+            "info": message
+        })
+    else:
+        logging.warning("No chat_publish_progress function provided!")
+        logging.log(logging.INFO, message)
+    
+
+    if request_id in shared_queues and not shared_queues[request_id].empty():
+        human_input = shared_queues[request_id].get()
+        if human_input:
+            if human_input == "TERMINATE":
+                logging.info("Terminating conversation")
+                raise Exception("Conversation terminated by user")
+            elif human_input == "PAUSE":
+                logging.info("Pausing conversation")
+                pause_start = time.time()
+                while time.time() - pause_start < 60*15:  # 15 minutes pause timeout
+                    time.sleep(10)
+                    new_input = check_for_human_input(request_id)
+                    if new_input:
+                        logging.info(f"Resuming conversation with human input: {new_input}")
+                        return logged_send(sender, original_send, new_input, recipient, request_reply, silent)
+                logging.info("Pause timeout, ending conversation")
+                raise Exception("Conversation ended due to pause timeout")
+            logging.info(f"Human input to {recipient.name}: {human_input}")
+            return original_send(human_input, recipient, request_reply, silent)
+
+    logging.info(f"Message from {sender.name} to {recipient.name}: {message}")
+
+    return original_send(message, recipient, request_reply, silent)
+
+
+def process_message(original_request_message_data, original_request_message_data_obj):    
+    try:
+        all_messages = []
+        started_at = datetime.now()
+        request_id = original_request_message_data.get('requestId') or original_request_message_data.id
+        original_request_message = original_request_message_data['message']
+
+        shared_queues[request_id] = queue.Queue()
+        thread = threading.Thread(target=background_human_input_check, args=(request_id,))
+        thread.daemon = True
+        thread.start()
+
+        final_msg = process_message_safe(original_request_message_data, original_request_message_data_obj, original_request_message,  all_messages, request_id, started_at)
+
+        finalData = {
+            "requestId": request_id,
+            "requestMessage": original_request_message_data.get("message"),
+            "progress": 1,
+            "data": final_msg,
+            "contextId": original_request_message_data.get("contextId"),
+            "conversation": all_messages,
+            "createdAt": datetime.now(timezone.utc).isoformat(),
+            "insertionTime": original_request_message_data_obj.insertion_time.astimezone(timezone.utc).isoformat(),
+            "startedAt": started_at.astimezone(timezone.utc).isoformat(),
+        }       
+
+        publish_request_progress(finalData)
+        store_in_mongo(finalData)
+
+
+    except Exception as e:
+        logging.error(f"Error processing message: {str(e)}")
+        try:
+            if request_id:
+                publish_request_progress({
+                    "requestId": request_id,
+                    "progress": 1,
+                    "error": str(e),
+                    "data": str(e),
+                })
+                store_in_mongo({
+                    "requestId": request_id,
+                    "requestMessage": original_request_message_data.get("message"),
+                    "progress": 1,
+                    "error": str(e),
+                    "data": str(e),
+                    "contextId": original_request_message_data.get("contextId"),
+                    "conversation": all_messages,
+                    "createdAt": datetime.now(timezone.utc).isoformat(),
+                    "insertionTime": original_request_message_data_obj.insertion_time.astimezone(timezone.utc).isoformat(),
+                    "startedAt": started_at.astimezone(timezone.utc).isoformat(),
+                })
+        except Exception as e:
+            logging.error(f"Error processing message finish publish&store: {str(e)}")
+
+
+
+def process_message_safe(original_request_message_data, original_request_message_data_obj, original_request_message,  all_messages, request_id, started_at):
+    config_list = config_list_from_json(env_or_file="OAI_CONFIG_LIST")
+    llm_config = {
+        "config_list": config_list,
+        "base_url": os.environ.get("CORTEX_API_BASE_URL"),
+        "api_key": os.environ.get("CORTEX_API_KEY"),
+        "cache_seed": None,
+        "timeout": 600 * 2
+    }
+
+    total_messages = 30 # set this for updates % progress's max just a guess
+    message_count = 0 # updates % progress 
+
+    def chat_publish_progress(data):
+        nonlocal message_count
+        message = data.get("message") or data.get("info") or data.get("content")
+        message_count += 1
+        progress = min(message_count / total_messages, 1)
+        publish_request_progress({
+            "requestId": request_id,
+            "progress": progress,
+            "info": message
+        })
+
+    def chat(*args,**kwargs):
+        nonlocal llm_config, request_id, chat_publish_progress, all_messages, original_request_message_data, original_request_message_data_obj, original_request_message
+        def get_arg_or_kwarg(name, pos, args, kwargs):
+            if args and kwargs.get(name): 
+                logging.warning(f"Both positional argument and keyword argument given for {name}, using keyword argument")
+            if kwargs.get(name):
+                return kwargs.get(name)
+            if len(args) > pos:
+                return args[pos]
+            return None
+
+        kwargs["prompt"] = get_arg_or_kwarg("prompt", 0, args, kwargs)
+        kwargs["message"] = get_arg_or_kwarg("message", 1, args, kwargs)
+        kwargs["llm_config"] = llm_config
+        kwargs["request_id"] = request_id
+        kwargs["chat_publish_progress"] = chat_publish_progress
+        kwargs["all_messages"] = all_messages
+        kwargs["original_request_message_data"] = original_request_message_data
+        kwargs["original_request_message_data_obj"] = original_request_message_data_obj
+        kwargs["original_request_message"] = original_request_message
+
+        return chat_with_agents(**kwargs)
+
+
+    preparer = AssistantAgent("preparer", llm_config=llm_config, system_message=prompts.get("PLANNER_SYSTEM_MESSAGE"))
+    prepared_plan = preparer.generate_reply(messages=[{"content": original_request_message, "role":"user"}])
+
+    helper_decider = AssistantAgent("helper_decider", llm_config=llm_config, system_message=prompts.get("HELPER_DECIDER_SYSTEM_MESSAGE"))
+    helper_decider_result = helper_decider.generate_reply(messages=[{"content": original_request_message, "role":"user"}])
+
+    try:
+        helper_decider_result = json.loads(helper_decider_result)
+        logging.info(f"Helper decider result: {helper_decider_result}")
+    except Exception as e:
+        logging.error(f"Error parsing helper decider result: {e}")
+        helper_decider_result = {}
+
+    context = ""
+    if helper_decider_result.get("bing_search"):
+        bing_search_message = f"Search Bing for more information on the task: {original_request_message}, prepared draft plan to solve task: {prepared_plan}"
+        result = chat(prompts.get("BING_SEARCH_PROMPT"), bing_search_message)
+        context += f"\n\nBing search results: {result}"
+
+    if helper_decider_result.get("cognitive_search"):
+        cognitive_search_message = f"Search cognitive index for more information on the task: {original_request_message}."
+        result = chat(prompts.get("COGNITIVE_SEARCH_PROMPT"), cognitive_search_message)
+        context += f"\n\nCognitive search results: {result}"
+
+
+    context = process_helper_results(helper_decider_result, original_request_message, context, chat) 
+
+    context_message = ""
+    if context:
+        context_message = f"\n\nHere is some data from search results and helpful stuff already collected and worked on, use if helpful:\n{context}\n\n"
+
+
+    check_message = f"""
+Task: \n{original_request_message}\n\n
+Context to check if task can be considered completed: {context_message}\n\n
+    """
+
+    task_completion_checker = AssistantAgent("task_completion_checker", llm_config=llm_config, system_message=TASK_COMPLETE_CHECKER_SYSTEM_MESSAGE)
+    check_result = task_completion_checker.generate_reply(messages=[{"content": check_message, "role":"user"}])
+
+    chat_result = None
+    if check_result != "DONE":
+        message = f"""
+Your task is to complete the following: \n{original_request_message}\n\n"
+Here is a draft plan to solve the task: \n{prepared_plan}\n\n 
+{context_message}
+You don't have to follow the plan, it's just a suggestion.
+Do your best to complete the task, user expects you to continue original task request conversation.
+"""
+        chat_result = chat(prompts.get("GENERIC_ASSISTANT_SYSTEM_MESSAGE"), message, return_type="chat_result")
+
+    presenter = AssistantAgent("presenter", llm_config=llm_config, system_message=prompts.get("PRESENTER_SYSTEM_MESSAGE"))
+    if chat_result is not None:
+        presenter_messages_context = "\n\n".join([msg['content'] for msg in chat_result.chat_history])
+    else:
+        presenter_messages_context = context_message
+    presenter_message = f"""
+Here is everything done in order to complete the task: {presenter_messages_context}\n\n 
+Original task was: {original_request_message}\n\n
+Reply to it with task result, do not forget that user expects you continue original task request conversation:\n\n
+"""
+
+    presenter_result = presenter.generate_reply(messages=[{"content": presenter_message, "role":"user"}])
+    
+    final_msg = presenter_result
+
+
+
+    zip_url = None  # TODO: Implement if needed
+    if zip_url:
+        final_msg += f"\n\n[Download all files of this task]({zip_url})"
+    
+    print(f"Task completed, task: {original_request_message}, result: {final_msg}")
+    logging.info(f"Task completed, task:\n{original_request_message},\nresult: {final_msg}")
+    return final_msg

package/helper-apps/cortex-autogen/agents_extra.py

@@ -0,0 +1,14 @@
+from config import prompts
+from datetime import datetime
+
+def process_helper_results(helper_decider_result, original_request_message, context, chat):
+    def add_to_context(result, prefix):
+        nonlocal context
+        context += f"\n\n{prefix}: {result}"
+
+    if helper_decider_result.get("sql"):
+        sql_message = f"Use SQL to help solving task, provide any related data and code that may help: {original_request_message}."
+        result = chat(prompts.get("SQL_PROMPT"), sql_message, return_type="all_as_str")
+        add_to_context(result, "SQL results")
+
+    return context

package/helper-apps/cortex-autogen/config.py

@@ -0,0 +1,18 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+AZURE_STORAGE_CONNECTION_STRING = os.environ["AZURE_STORAGE_CONNECTION_STRING"]
+HUMAN_INPUT_QUEUE_NAME = os.environ.get("HUMAN_INPUT_QUEUE_NAME", "autogen-human-input-queue")
+REDIS_CONNECTION_STRING = os.environ['REDIS_CONNECTION_STRING']
+REDIS_CHANNEL = 'requestProgress'
+AZURE_BLOB_CONTAINER = os.environ.get("AZURE_BLOB_CONTAINER", "autogen-uploads")
+
+
+# Prompts
+import prompts
+import prompts_extra
+
+prompts = {**prompts.__dict__, **prompts_extra.__dict__}
+

package/helper-apps/cortex-autogen/data_operations.py

@@ -0,0 +1,29 @@
+from azure.storage.queue import QueueClient
+import pymongo
+import os
+import logging
+import json
+import base64
+from config import AZURE_STORAGE_CONNECTION_STRING, HUMAN_INPUT_QUEUE_NAME
+
+human_input_queue_client = QueueClient.from_connection_string(AZURE_STORAGE_CONNECTION_STRING, HUMAN_INPUT_QUEUE_NAME)
+
+def store_in_mongo(data):
+    try:
+        if 'MONGO_URI' in os.environ:
+            client = pymongo.MongoClient(os.environ['MONGO_URI'])
+            collection = client.get_default_database()[os.environ.get('MONGO_COLLECTION_NAME', 'autogenruns')]
+            collection.insert_one(data)
+        else:
+            logging.warning("MONGO_URI not found in environment variables")
+    except Exception as e:
+        logging.error(f"An error occurred while storing data in MongoDB: {str(e)}")
+
+def check_for_human_input(request_id):
+    messages = human_input_queue_client.receive_messages()
+    for message in messages:
+        content = json.loads(base64.b64decode(message.content).decode('utf-8'))
+        if content['codeRequestId'] == request_id:
+            human_input_queue_client.delete_message(message)
+            return content['text']
+    return None

package/helper-apps/cortex-autogen/function_app.py

@@ -4,14 +4,17 @@ import json
 from azure.storage.queue import QueueClient
 import os
 import redis
-from myautogen import process_message
+from agents import process_message
 import subprocess
 import sys
+import config 
+
+logging.getLogger().setLevel(logging.WARNING)
 
 def install_packages():
     subprocess.check_call([sys.executable, "-m", "pip", "install", "-r", "requirements.txt"])
 
-install_packages()
+# install_packages()
 
 app = func.FunctionApp()
 
@@ -31,6 +34,6 @@ def queue_trigger(msg: func.QueueMessage):
         if "requestId" not in message_data:
             message_data['requestId'] = msg.id
         process_message(message_data, msg)
-
+        
     except Exception as e:
         logging.error(f"Error processing message: {str(e)}")

package/helper-apps/cortex-autogen/main.py

@@ -2,8 +2,9 @@ import os
 from azure.storage.queue import QueueClient
 import base64
 import json
-from myautogen import process_message
 import time
+from agents import process_message
+
 
 def main():
     print("Starting message processing loop")
@@ -13,9 +14,8 @@ def main():
     queue_client = QueueClient.from_connection_string(connection_string, queue_name)
     
     attempts = 0
-    max_attempts = 100
+    max_attempts = 1000
     
-
     while attempts < max_attempts:
         messages = queue_client.receive_messages(messages_per_page=1)
         
@@ -32,7 +32,7 @@ def main():
             attempts += 1
             time.sleep(1)  # Wait for 1 second before checking again
 
-    print("No messages received after 100 attempts. Exiting.")
+    print(f"No messages received after {max_attempts} attempts. Exiting.")
 
 if __name__ == "__main__":
     main()