@aj-archipelago/cortex 1.3.4 → 1.3.6

package/helper-apps/cortex-autogen/agents.py

@@ -19,6 +19,8 @@ import threading
 import shutil
 
 human_input_queues = {}
+human_input_text_queues = {}
+request_stored_message_queues = {}
 def background_human_input_check(request_id):
     while True:
         human_input = check_for_human_input(request_id)
@@ -26,8 +28,20 @@ def background_human_input_check(request_id):
             human_input_queues[request_id].put(human_input)
             if human_input in ["TERMINATE", "PAUSE"]:
                 break
+            else:
+                if not human_input_text_queues.get(request_id):
+                    human_input_text_queues[request_id] = queue.Queue()
+                human_input_text_queues[request_id].put(human_input)
         time.sleep(1)
 
+
+def get_message_with_user_input(message, request_id):
+    human_input_text = ""
+    if human_input_text_queues.get(request_id):
+        while not human_input_text_queues[request_id].empty():
+            human_input_text += " " + human_input_text_queues[request_id].get()
+    return message + human_input_text
+
 def get_request_temp_dir(request_id):
     if not request_id:
         logging.warning("No request_id provided!")
@@ -100,6 +114,7 @@ def chat_with_agents(**kwargs):
     original_request_message = kwargs.pop("original_request_message", None)
     original_request_message_data = kwargs.pop("original_request_message_data", None)
 
+
     llm_config = kwargs.pop("llm_config", None)
     request_id = kwargs.pop("request_id", None)
     chat_publish_progress = kwargs.pop("chat_publish_progress", None)
@@ -137,13 +152,14 @@ def chat_with_agents(**kwargs):
         assistant.send = create_send_function(assistant)
         user_proxy.send = create_send_function(user_proxy)
 
+        message_with_possible_human_input = get_message_with_user_input(message, request_id)
+
         chat_result = user_proxy.initiate_chat(
             assistant, 
-            message=message, 
+            message=message_with_possible_human_input, 
         )
 
 
-
     code_msg = find_code_message(all_messages)
     if code_msg:
         try:
@@ -155,12 +171,20 @@ def chat_with_agents(**kwargs):
             index_message({
                 "requestId": request_id,
                 "content":corrector_result, #code_msg,
-                "task": original_request_message,
+                "task": get_message_with_user_input(original_request_message,request_id),
                 "contextId": original_request_message_data.get("contextId"),
             })
         except Exception as e:
             logging.error(f"Error extracting code corrector result: {e}")
 
+    try:
+        request_stored_message_queues[request_id].put(all_messages[-2]["message"] or all_messages[-2]["content"])
+        request_stored_message_queues[request_id].put(all_messages[-1]["message"] or all_messages[-1]["content"])
+    except Exception as e:
+        logging.error(f"Error storing messages in queue: {e}")
+    
+
+
     if return_type == "chat_history":
         return chat_result.chat_history
     if return_type == "chat_result":
@@ -175,7 +199,6 @@ def chat_with_agents(**kwargs):
     return chat_result 
 
 
-
 def logged_send(sender, original_send, message, recipient, request_reply=None, silent=True, request_id=None, chat_publish_progress=None, all_messages=None):
     if not message:
         logging.info("Empty message, skipping!")
@@ -214,15 +237,19 @@ def logged_send(sender, original_send, message, recipient, request_reply=None, s
                         return logged_send(sender, original_send, new_input, recipient, request_reply, silent)
                 logging.info("Pause timeout, ending conversation")
                 raise Exception("Conversation ended due to pause timeout")
+
+            #if not terminate or pause, then it's text input from human
             logging.info(f"Human input to {recipient.name}: {human_input}")
-            return original_send(human_input, recipient, request_reply, silent)
+            #need to update original message with human input
+            new_input = message + human_input
+            return original_send(new_input, recipient, request_reply, silent)
 
     logging.info(f"Message from {sender.name} to {recipient.name}: {message}")
 
     return original_send(message, recipient, request_reply, silent)
 
 
-def process_message(original_request_message_data, original_request_message_data_obj):    
+def process_message(original_request_message_data, original_request_message_data_obj, first_run=True):    
     try:
         all_messages = []
         started_at = datetime.now()
@@ -230,9 +257,15 @@ def process_message(original_request_message_data, original_request_message_data
         original_request_message = original_request_message_data['message']
 
         human_input_queues[request_id] = queue.Queue()
-        thread = threading.Thread(target=background_human_input_check, args=(request_id,))
-        thread.daemon = True
-        thread.start()
+        human_input_text_queues[request_id] = queue.Queue()
+        if not request_stored_message_queues.get(request_id):
+            request_stored_message_queues[request_id] = queue.Queue()
+        request_stored_message_queues[request_id].put(original_request_message)
+
+        if first_run:
+            thread = threading.Thread(target=background_human_input_check, args=(request_id,))
+            thread.daemon = True
+            thread.start()
 
         final_msg = process_message_safe(original_request_message_data, original_request_message_data_obj, original_request_message,  all_messages, request_id, started_at)
 
@@ -252,6 +285,44 @@ def process_message(original_request_message_data, original_request_message_data
         publish_request_progress(finalData)
         store_in_mongo(finalData)
 
+        #wait for any human input before terminating
+        #if you receive human input start the conversation again
+        for i in range(31*6): # 30+1 minutes
+            if human_input_queues[request_id].empty():
+                time.sleep(1)
+            else:
+                human_input = human_input_queues[request_id].get()
+                if human_input:
+                    logging.info(f"Human input to assistant: {human_input}")
+                    #update request with human input
+                    new_message_data = original_request_message_data.copy()
+
+                    old_task = original_request_message_data.get("message")
+
+                    #get request_stored_message_queues
+                    old_messages = []
+                    if request_stored_message_queues.get(request_id):
+                        while not request_stored_message_queues[request_id].empty():
+                            old_messages.append(request_stored_message_queues[request_id].get())
+
+
+                    #convert to text, limit to max 2000 characters, keep most recent
+                    old_messages_text = "\n".join(old_messages)
+                    old_messages_text = old_messages_text[-2000:]
+
+
+                    new_message_data['message'] = f"NEW TASK: {human_input}\n\nPREV TASK: {old_task} STUFF DONE IN PREV TASK: {old_messages_text}\n\n{final_msg}\n\n"
+                    new_message_data['keywords'] = ''
+                    # new_message_data_obj = original_request_message_data_obj.copy()
+                    # new_message_data_obj['message'] = new_message_data['message']
+
+
+
+                    process_message(new_message_data, original_request_message_data_obj, first_run=False)
+                    return
+
+        logging.info(f"Task completed, task:\n{get_message_with_user_input(original_request_message,request_id)},\nresult: {final_msg}")
+
 
     except Exception as e:
         logging.error(f"Error processing message: {str(e)}")
@@ -292,7 +363,6 @@ def process_message(original_request_message_data, original_request_message_data
             logging.error(f"Error cleaning up: {str(e)}")
             
 
-
 def process_message_safe(original_request_message_data, original_request_message_data_obj, original_request_message,  all_messages, request_id, started_at):
     config_list = config_list_from_json(env_or_file="OAI_CONFIG_LIST")
     llm_config = {
@@ -342,10 +412,10 @@ def process_message_safe(original_request_message_data, original_request_message
 
 
     preparer = AssistantAgent("preparer", llm_config=llm_config, system_message=prompts.get("PLANNER_SYSTEM_MESSAGE"))
-    prepared_plan = preparer.generate_reply(messages=[{"content": original_request_message, "role":"user"}])
+    prepared_plan = preparer.generate_reply(messages=[{"content": get_message_with_user_input(original_request_message,request_id), "role":"user"}])
 
     helper_decider = AssistantAgent("helper_decider", llm_config=llm_config, system_message=prompts.get("HELPER_DECIDER_SYSTEM_MESSAGE"))
-    helper_decider_result = helper_decider.generate_reply(messages=[{"content": original_request_message, "role":"user"}])
+    helper_decider_result = helper_decider.generate_reply(messages=[{"content": get_message_with_user_input(original_request_message,request_id), "role":"user"}])
 
     try:
         helper_decider_result = json.loads(helper_decider_result)
@@ -361,17 +431,17 @@ def process_message_safe(original_request_message_data, original_request_message
         context += f"\n#SECTION_OF_OLD_TASK_CODE_INFO_START:\nHere's code/info from old-tasks that might help:\n{search_index(code_keywords)}\n#SECTION_OF_OLD_TASK_CODE_INFO_END\n"
 
     if helper_decider_result.get("bing_search"):
-        bing_search_message = f"Search Bing for more information on the task: {original_request_message}, prepared draft plan to solve task: {prepared_plan}"
+        bing_search_message = f"Search Bing for more information on the task: {get_message_with_user_input(original_request_message,request_id)}, prepared draft plan to solve task: {prepared_plan}"
         result = chat(prompts.get("BING_SEARCH_PROMPT"), bing_search_message)
         context += f"\n\nBing search results: {result}"
 
     if helper_decider_result.get("cognitive_search"):
-        cognitive_search_message = f"Search cognitive index for more information on the task: {original_request_message}."
+        cognitive_search_message = f"Search cognitive index for more information on the task: {get_message_with_user_input(original_request_message,request_id)}."
         result = chat(prompts.get("COGNITIVE_SEARCH_PROMPT"), cognitive_search_message)
         context += f"\n\nCognitive search results: {result}"
 
 
-    context = process_helper_results(helper_decider_result, original_request_message, context, chat) 
+    context = process_helper_results(helper_decider_result, get_message_with_user_input(original_request_message,request_id), context, chat) 
 
     context_message = ""
     if context:
@@ -379,7 +449,7 @@ def process_message_safe(original_request_message_data, original_request_message
 
 
     check_message = f"""
-Task: \n{original_request_message}\n\n
+Task: \n{get_message_with_user_input(original_request_message,request_id)}\n\n
 Context to check if task can be considered completed: {context_message}\n\n
     """
 
@@ -389,7 +459,7 @@ Context to check if task can be considered completed: {context_message}\n\n
     chat_result = None
     if check_result != "DONE":
         message = f"""
-Your task is to complete the following: \n{original_request_message}\n\n"
+Your task is to complete the following: \n{get_message_with_user_input(original_request_message,request_id)}\n\n"
 Here is a draft plan to solve the task: \n{prepared_plan}\n\n 
 {context_message}
 You don't have to follow the plan, it's just a suggestion.
@@ -404,7 +474,7 @@ Do your best to complete the task, user expects you to continue original task re
         presenter_messages_context = context_message
     presenter_message = f"""
 Here is everything done in order to complete the task: {presenter_messages_context}\n\n 
-Original task was: {original_request_message}\n\n
+Original task was: {get_message_with_user_input(original_request_message,request_id)}\n\n
 Reply to it with task result, do not forget that user expects you continue original task request conversation:\n\n
 """
 
@@ -418,6 +488,6 @@ Reply to it with task result, do not forget that user expects you continue origi
         final_msg += f"\n\n[Download all files of this task]({zip_url})"
     
     
-    print(f"Task completed, task:\n{original_request_message},\nresult: {final_msg}")
-    logging.info(f"Task completed, task:\n{original_request_message},\nresult: {final_msg}")
+    print(f"Task completed, task:\n{get_message_with_user_input(original_request_message,request_id)},\nresult: {final_msg}")
+    logging.info(f"Task completed, task:\n{get_message_with_user_input(original_request_message,request_id)},\nresult: {final_msg}")
     return final_msg

package/helper-apps/cortex-realtime-voice-server/.env.sample

@@ -0,0 +1,6 @@
+CORTEX_API_KEY=...
+CORTEX_DEV_API_KEY=...
+OPENAI_API_KEY=...
+CORS_HOSTS='["http://localhost:3000"]'
+PORT=8081
+VOICE_LIB_DEBUG=false

package/helper-apps/cortex-realtime-voice-server/README.md

@@ -0,0 +1,22 @@
+# cortex-realtime-voice
+
+To install dependencies:
+
+```bash
+bun install
+```
+
+To run:
+
+```bash
+cd client
+bun run build
+cd ..
+bun run start
+```
+
+To run in production:
+
+```bash
+bun run start:prod
+```

package/helper-apps/cortex-realtime-voice-server/client/index.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Cortex Realtime Voice</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/index.tsx"></script>
+  </body>
+</html> 

package/helper-apps/cortex-realtime-voice-server/client/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "client",
+  "version": "0.1.0",
+  "private": true,
+  "dependencies": {
+    "@emotion/react": "^11.14.0",
+    "@emotion/styled": "^11.14.0",
+    "@mui/icons-material": "^6.2.0",
+    "@mui/material": "^6.2.0",
+    "@tailwindcss/typography": "^0.5.15",
+    "katex": "^0.16.15",
+    "react": "^19.0.0",
+    "react-dom": "^19.0.0",
+    "react-markdown": "^9.0.1",
+    "react-spinners": "0.15.0",
+    "react-syntax-highlighter": "^15.6.1",
+    "rehype-katex": "^7.0.1",
+    "rehype-raw": "^7.0.0",
+    "remark-gfm": "^4.0.0",
+    "remark-math": "^6.0.0",
+    "socket.io-client": "4.8.1",
+    "typescript": "^4.4.2",
+    "web-vitals": "^2.1.0"
+  },
+  "devDependencies": {
+    "@babel/plugin-proposal-private-property-in-object": "7.21.11",
+    "@testing-library/jest-dom": "^5.14.1",
+    "@testing-library/react": "^13.0.0",
+    "@testing-library/user-event": "^13.2.1",
+    "@types/jest": "^27.0.1",
+    "@types/node": "^16.7.13",
+    "@types/react": "^18.0.0",
+    "@types/react-dom": "^18.0.0",
+    "@types/react-syntax-highlighter": "^15.5.13",
+    "@vitejs/plugin-react": "^4.3.4",
+    "autoprefixer": "^10.4.20",
+    "postcss": "^8.4.49",
+    "tailwindcss": "^3.4.17",
+    "vite": "^6.0.5"
+  },
+  "scripts": {
+    "start": "vite",
+    "dev": "vite",
+    "build": "vite build",
+    "preview": "vite preview"
+  },
+  "eslintConfig": {
+    "extends": [
+      "react-app",
+      "react-app/jest"
+    ]
+  },
+  "browserslist": {
+    "production": [
+      ">0.2%",
+      "not dead",
+      "not op_mini all"
+    ],
+    "development": [
+      "last 1 chrome version",
+      "last 1 firefox version",
+      "last 1 safari version"
+    ]
+  }
+}

package/helper-apps/cortex-realtime-voice-server/client/postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+} 

package/helper-apps/cortex-realtime-voice-server/client/public/index.html

@@ -0,0 +1,43 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta name="theme-color" content="#000000" />
+    <meta
+      name="description"
+      content="Chat with the Cortex Realtime Voice Server"
+    />
+    <link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />
+    <!--
+      manifest.json provides metadata used when your web app is installed on a
+      user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
+    -->
+    <link rel="manifest" href="%PUBLIC_URL%/manifest.json" />
+    <!--
+      Notice the use of %PUBLIC_URL% in the tags above.
+      It will be replaced with the URL of the `public` folder during the build.
+      Only files inside the `public` folder can be referenced from the HTML.
+
+      Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
+      work correctly both with client-side routing and a non-root public URL.
+      Learn how to configure a non-root public URL by running `npm run build`.
+    -->
+    <title>Cortex Realtime Voice</title>
+  </head>
+  <body>
+    <noscript>You need to enable JavaScript to run this app.</noscript>
+    <div id="root"></div>
+    <!--
+      This HTML file is a template.
+      If you open it directly in the browser, you will see an empty page.
+
+      You can add webfonts, meta tags, or analytics to this file.
+      The build step will place the bundled scripts into the <body> tag.
+
+      To begin the development, run `npm start` or `yarn start`.
+      To create a production bundle, use `npm run build` or `yarn build`.
+    -->
+  </body>
+</html>

package/helper-apps/cortex-realtime-voice-server/client/public/manifest.json

@@ -0,0 +1,25 @@
+{
+  "short_name": "React App",
+  "name": "Create React App Sample",
+  "icons": [
+    {
+      "src": "favicon.ico",
+      "sizes": "64x64 32x32 24x24 16x16",
+      "type": "image/x-icon"
+    },
+    {
+      "src": "logo192.png",
+      "type": "image/png",
+      "sizes": "192x192"
+    },
+    {
+      "src": "logo512.png",
+      "type": "image/png",
+      "sizes": "512x512"
+    }
+  ],
+  "start_url": ".",
+  "display": "standalone",
+  "theme_color": "#000000",
+  "background_color": "#ffffff"
+}

package/helper-apps/cortex-realtime-voice-server/client/public/robots.txt

@@ -0,0 +1,3 @@
+# https://www.robotstxt.org/robotstxt.html
+User-agent: *
+Disallow:

package/helper-apps/cortex-realtime-voice-server/client/src/App.test.tsx

@@ -0,0 +1,9 @@
+import React from 'react';
+import { render, screen } from '@testing-library/react';
+import App from './App';
+
+test('renders learn react link', () => {
+  render(<App />);
+  const linkElement = screen.getByText(/learn react/i);
+  expect(linkElement).toBeInTheDocument();
+});

package/helper-apps/cortex-realtime-voice-server/client/src/App.tsx

@@ -0,0 +1,126 @@
+import {useEffect, useState} from 'react';
+import ClipLoader from "react-spinners/ClipLoader";
+import Chat from "./chat/Chat";
+import {SettingsModal} from "./SettingsModal";
+import { SettingsData } from "./SettingsModal";
+import type { Voice } from '../../src/realtime/realtimeTypes';
+
+function App() {
+  const [loading, setLoading] = useState(true);
+  const [userName, setUserName] = useState('');
+  const [userId, setUserId] = useState('');
+  const [aiName, setAiName] = useState('Jarvis');
+  const [settingsOpen, setSettingsOpen] = useState(false);
+  const [language, setLanguage] = useState('en');
+  const [aiMemorySelfModify, setAiMemorySelfModify] = useState(false);
+  const [aiStyle, setAiStyle] = useState('Anthropic');
+  const [voice, setVoice] = useState('alloy' as Voice);
+
+  const onCloseSettings = () => setSettingsOpen(false);
+  const onSaveSettings = (settings: SettingsData) => {
+    console.log('Saving settings', settings);
+    setUserName(settings.userName);
+    localStorage.setItem('userName', settings.userName);
+
+    let newUserId = settings.userId;
+    if (!newUserId || newUserId.length === 0) {
+      newUserId = Math.random().toString(36).substring(7);
+    }
+    setUserId(newUserId);
+    localStorage.setItem('userId', newUserId);
+    
+    setAiName(settings.aiName);
+    localStorage.setItem('aiName', settings.aiName);
+    
+    setLanguage(settings.language);
+    localStorage.setItem('language', settings.language);
+    
+    setAiMemorySelfModify(settings.aiMemorySelfModify);
+    localStorage.setItem('aiMemorySelfModify', String(settings.aiMemorySelfModify));
+    
+    setAiStyle(settings.aiStyle);
+    localStorage.setItem('aiStyle', settings.aiStyle);
+    
+    setVoice(settings.voice as Voice);
+    localStorage.setItem('voice', settings.voice);
+  };
+
+  useEffect(() => {
+    const name = localStorage.getItem('userName');
+    if (name) {
+      setUserName(localStorage.getItem('userName') as string);
+    } else {
+      setSettingsOpen(true);
+    }
+    const id = localStorage.getItem('userId');
+    if (id) {
+      setUserId(localStorage.getItem('userId') as string);
+    } else {
+      setSettingsOpen(true);
+    }
+    const ai = localStorage.getItem('aiName');
+    if (ai) {
+      setAiName(localStorage.getItem('aiName') as string);
+    } else {
+      setSettingsOpen(true);
+    }
+    const savedLanguage = localStorage.getItem('language');
+    if (savedLanguage) setLanguage(savedLanguage);
+    
+    const savedAiMemorySelfModify = localStorage.getItem('aiMemorySelfModify');
+    if (savedAiMemorySelfModify) setAiMemorySelfModify(savedAiMemorySelfModify === 'true');
+    
+    const savedAiStyle = localStorage.getItem('aiStyle');
+    if (savedAiStyle) setAiStyle(savedAiStyle);
+    
+    const savedVoice = localStorage.getItem('voice');
+    if (savedVoice) setVoice(savedVoice as Voice);
+    
+    setLoading(false);
+  }, []);
+
+  if (loading) {
+    return (
+      <div className="min-h-screen bg-gradient-to-b from-gray-900 via-gray-800 to-gray-900 flex justify-center items-center">
+        <ClipLoader color={'#0EA5E9'} loading={true} size={150}/>
+      </div>
+    );
+  }
+
+  return (
+    <div className="h-screen overflow-hidden bg-gradient-to-b from-gray-900 via-gray-800 to-gray-900">
+      <button 
+        className="fixed top-4 right-4 z-10 p-2 text-gray-400 hover:text-cyan-400 transition-colors duration-200"
+        onClick={() => setSettingsOpen(true)}
+        aria-label="Settings"
+      >
+        ⚙️
+      </button>
+      
+      {userName && userName.length > 0 && (
+        <Chat 
+          userId={userId} 
+          userName={userName} 
+          aiName={aiName}
+          language={language}
+          aiMemorySelfModify={aiMemorySelfModify}
+          aiStyle={aiStyle}
+          voice={voice}
+        />
+      )}
+      <SettingsModal 
+        aiName={aiName}
+        userName={userName}
+        userId={userId}
+        voice={voice}
+        aiStyle={aiStyle}
+        language={language}
+        isOpen={settingsOpen}
+        onClose={onCloseSettings}
+        onSave={onSaveSettings}
+      />
+    </div>
+  );
+}
+
+export default App;