@aj-archipelago/cortex 1.3.5 → 1.3.7

package/helper-apps/cortex-autogen/agents.py

@@ -20,6 +20,7 @@ import shutil
 
 human_input_queues = {}
 human_input_text_queues = {}
+request_stored_message_queues = {}
 def background_human_input_check(request_id):
     while True:
         human_input = check_for_human_input(request_id)
@@ -176,6 +177,14 @@ def chat_with_agents(**kwargs):
         except Exception as e:
             logging.error(f"Error extracting code corrector result: {e}")
 
+    try:
+        request_stored_message_queues[request_id].put(all_messages[-2]["message"] or all_messages[-2]["content"])
+        request_stored_message_queues[request_id].put(all_messages[-1]["message"] or all_messages[-1]["content"])
+    except Exception as e:
+        logging.error(f"Error storing messages in queue: {e}")
+    
+
+
     if return_type == "chat_history":
         return chat_result.chat_history
     if return_type == "chat_result":
@@ -249,6 +258,9 @@ def process_message(original_request_message_data, original_request_message_data
 
         human_input_queues[request_id] = queue.Queue()
         human_input_text_queues[request_id] = queue.Queue()
+        if not request_stored_message_queues.get(request_id):
+            request_stored_message_queues[request_id] = queue.Queue()
+        request_stored_message_queues[request_id].put(original_request_message)
 
         if first_run:
             thread = threading.Thread(target=background_human_input_check, args=(request_id,))
@@ -275,7 +287,7 @@ def process_message(original_request_message_data, original_request_message_data
 
         #wait for any human input before terminating
         #if you receive human input start the conversation again
-        for i in range(16*6): # 15+1 minutes
+        for i in range(31*6): # 30+1 minutes
             if human_input_queues[request_id].empty():
                 time.sleep(1)
             else:
@@ -284,11 +296,28 @@ def process_message(original_request_message_data, original_request_message_data
                     logging.info(f"Human input to assistant: {human_input}")
                     #update request with human input
                     new_message_data = original_request_message_data.copy()
-                    new_message_data['message'] = human_input
+
+                    old_task = original_request_message_data.get("message")
+
+                    #get request_stored_message_queues
+                    old_messages = []
+                    if request_stored_message_queues.get(request_id):
+                        while not request_stored_message_queues[request_id].empty():
+                            old_messages.append(request_stored_message_queues[request_id].get())
+
+
+                    #convert to text, limit to max 2000 characters, keep most recent
+                    old_messages_text = "\n".join(old_messages)
+                    old_messages_text = old_messages_text[-2000:]
+
+
+                    new_message_data['message'] = f"NEW TASK: {human_input}\n\nPREV TASK: {old_task} STUFF DONE IN PREV TASK: {old_messages_text}\n\n{final_msg}\n\n"
                     new_message_data['keywords'] = ''
                     # new_message_data_obj = original_request_message_data_obj.copy()
                     # new_message_data_obj['message'] = new_message_data['message']
 
+
+
                     process_message(new_message_data, original_request_message_data_obj, first_run=False)
                     return
 

package/helper-apps/cortex-realtime-voice-server/.env.sample

@@ -0,0 +1,6 @@
+CORTEX_API_KEY=...
+CORTEX_DEV_API_KEY=...
+OPENAI_API_KEY=...
+CORS_HOSTS='["http://localhost:3000"]'
+PORT=8081
+VOICE_LIB_DEBUG=false

package/helper-apps/cortex-realtime-voice-server/README.md

@@ -0,0 +1,22 @@
+# cortex-realtime-voice
+
+To install dependencies:
+
+```bash
+bun install
+```
+
+To run:
+
+```bash
+cd client
+bun run build
+cd ..
+bun run start
+```
+
+To run in production:
+
+```bash
+bun run start:prod
+```

package/helper-apps/cortex-realtime-voice-server/client/index.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Cortex Realtime Voice</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/index.tsx"></script>
+  </body>
+</html> 

package/helper-apps/cortex-realtime-voice-server/client/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "client",
+  "version": "0.1.0",
+  "private": true,
+  "dependencies": {
+    "@emotion/react": "^11.14.0",
+    "@emotion/styled": "^11.14.0",
+    "@mui/icons-material": "^6.2.0",
+    "@mui/material": "^6.2.0",
+    "@tailwindcss/typography": "^0.5.15",
+    "katex": "^0.16.15",
+    "react": "^19.0.0",
+    "react-dom": "^19.0.0",
+    "react-markdown": "^9.0.1",
+    "react-spinners": "0.15.0",
+    "react-syntax-highlighter": "^15.6.1",
+    "rehype-katex": "^7.0.1",
+    "rehype-raw": "^7.0.0",
+    "remark-gfm": "^4.0.0",
+    "remark-math": "^6.0.0",
+    "socket.io-client": "4.8.1",
+    "typescript": "^4.4.2",
+    "web-vitals": "^2.1.0"
+  },
+  "devDependencies": {
+    "@babel/plugin-proposal-private-property-in-object": "7.21.11",
+    "@testing-library/jest-dom": "^5.14.1",
+    "@testing-library/react": "^13.0.0",
+    "@testing-library/user-event": "^13.2.1",
+    "@types/jest": "^27.0.1",
+    "@types/node": "^16.7.13",
+    "@types/react": "^18.0.0",
+    "@types/react-dom": "^18.0.0",
+    "@types/react-syntax-highlighter": "^15.5.13",
+    "@vitejs/plugin-react": "^4.3.4",
+    "autoprefixer": "^10.4.20",
+    "postcss": "^8.4.49",
+    "tailwindcss": "^3.4.17",
+    "vite": "^6.0.5"
+  },
+  "scripts": {
+    "start": "vite",
+    "dev": "vite",
+    "build": "vite build",
+    "preview": "vite preview"
+  },
+  "eslintConfig": {
+    "extends": [
+      "react-app",
+      "react-app/jest"
+    ]
+  },
+  "browserslist": {
+    "production": [
+      ">0.2%",
+      "not dead",
+      "not op_mini all"
+    ],
+    "development": [
+      "last 1 chrome version",
+      "last 1 firefox version",
+      "last 1 safari version"
+    ]
+  }
+}

package/helper-apps/cortex-realtime-voice-server/client/postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+} 

package/helper-apps/cortex-realtime-voice-server/client/public/index.html

@@ -0,0 +1,43 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta name="theme-color" content="#000000" />
+    <meta
+      name="description"
+      content="Chat with the Cortex Realtime Voice Server"
+    />
+    <link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />
+    <!--
+      manifest.json provides metadata used when your web app is installed on a
+      user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
+    -->
+    <link rel="manifest" href="%PUBLIC_URL%/manifest.json" />
+    <!--
+      Notice the use of %PUBLIC_URL% in the tags above.
+      It will be replaced with the URL of the `public` folder during the build.
+      Only files inside the `public` folder can be referenced from the HTML.
+
+      Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
+      work correctly both with client-side routing and a non-root public URL.
+      Learn how to configure a non-root public URL by running `npm run build`.
+    -->
+    <title>Cortex Realtime Voice</title>
+  </head>
+  <body>
+    <noscript>You need to enable JavaScript to run this app.</noscript>
+    <div id="root"></div>
+    <!--
+      This HTML file is a template.
+      If you open it directly in the browser, you will see an empty page.
+
+      You can add webfonts, meta tags, or analytics to this file.
+      The build step will place the bundled scripts into the <body> tag.
+
+      To begin the development, run `npm start` or `yarn start`.
+      To create a production bundle, use `npm run build` or `yarn build`.
+    -->
+  </body>
+</html>

package/helper-apps/cortex-realtime-voice-server/client/public/manifest.json

@@ -0,0 +1,25 @@
+{
+  "short_name": "React App",
+  "name": "Create React App Sample",
+  "icons": [
+    {
+      "src": "favicon.ico",
+      "sizes": "64x64 32x32 24x24 16x16",
+      "type": "image/x-icon"
+    },
+    {
+      "src": "logo192.png",
+      "type": "image/png",
+      "sizes": "192x192"
+    },
+    {
+      "src": "logo512.png",
+      "type": "image/png",
+      "sizes": "512x512"
+    }
+  ],
+  "start_url": ".",
+  "display": "standalone",
+  "theme_color": "#000000",
+  "background_color": "#ffffff"
+}

package/helper-apps/cortex-realtime-voice-server/client/public/robots.txt

@@ -0,0 +1,3 @@
+# https://www.robotstxt.org/robotstxt.html
+User-agent: *
+Disallow:

package/helper-apps/cortex-realtime-voice-server/client/src/App.test.tsx

@@ -0,0 +1,9 @@
+import React from 'react';
+import { render, screen } from '@testing-library/react';
+import App from './App';
+
+test('renders learn react link', () => {
+  render(<App />);
+  const linkElement = screen.getByText(/learn react/i);
+  expect(linkElement).toBeInTheDocument();
+});

package/helper-apps/cortex-realtime-voice-server/client/src/App.tsx

@@ -0,0 +1,126 @@
+import {useEffect, useState} from 'react';
+import ClipLoader from "react-spinners/ClipLoader";
+import Chat from "./chat/Chat";
+import {SettingsModal} from "./SettingsModal";
+import { SettingsData } from "./SettingsModal";
+import type { Voice } from '../../src/realtime/realtimeTypes';
+
+function App() {
+  const [loading, setLoading] = useState(true);
+  const [userName, setUserName] = useState('');
+  const [userId, setUserId] = useState('');
+  const [aiName, setAiName] = useState('Jarvis');
+  const [settingsOpen, setSettingsOpen] = useState(false);
+  const [language, setLanguage] = useState('en');
+  const [aiMemorySelfModify, setAiMemorySelfModify] = useState(false);
+  const [aiStyle, setAiStyle] = useState('Anthropic');
+  const [voice, setVoice] = useState('alloy' as Voice);
+
+  const onCloseSettings = () => setSettingsOpen(false);
+  const onSaveSettings = (settings: SettingsData) => {
+    console.log('Saving settings', settings);
+    setUserName(settings.userName);
+    localStorage.setItem('userName', settings.userName);
+
+    let newUserId = settings.userId;
+    if (!newUserId || newUserId.length === 0) {
+      newUserId = Math.random().toString(36).substring(7);
+    }
+    setUserId(newUserId);
+    localStorage.setItem('userId', newUserId);
+    
+    setAiName(settings.aiName);
+    localStorage.setItem('aiName', settings.aiName);
+    
+    setLanguage(settings.language);
+    localStorage.setItem('language', settings.language);
+    
+    setAiMemorySelfModify(settings.aiMemorySelfModify);
+    localStorage.setItem('aiMemorySelfModify', String(settings.aiMemorySelfModify));
+    
+    setAiStyle(settings.aiStyle);
+    localStorage.setItem('aiStyle', settings.aiStyle);
+    
+    setVoice(settings.voice as Voice);
+    localStorage.setItem('voice', settings.voice);
+  };
+
+  useEffect(() => {
+    const name = localStorage.getItem('userName');
+    if (name) {
+      setUserName(localStorage.getItem('userName') as string);
+    } else {
+      setSettingsOpen(true);
+    }
+    const id = localStorage.getItem('userId');
+    if (id) {
+      setUserId(localStorage.getItem('userId') as string);
+    } else {
+      setSettingsOpen(true);
+    }
+    const ai = localStorage.getItem('aiName');
+    if (ai) {
+      setAiName(localStorage.getItem('aiName') as string);
+    } else {
+      setSettingsOpen(true);
+    }
+    const savedLanguage = localStorage.getItem('language');
+    if (savedLanguage) setLanguage(savedLanguage);
+    
+    const savedAiMemorySelfModify = localStorage.getItem('aiMemorySelfModify');
+    if (savedAiMemorySelfModify) setAiMemorySelfModify(savedAiMemorySelfModify === 'true');
+    
+    const savedAiStyle = localStorage.getItem('aiStyle');
+    if (savedAiStyle) setAiStyle(savedAiStyle);
+    
+    const savedVoice = localStorage.getItem('voice');
+    if (savedVoice) setVoice(savedVoice as Voice);
+    
+    setLoading(false);
+  }, []);
+
+  if (loading) {
+    return (
+      <div className="min-h-screen bg-gradient-to-b from-gray-900 via-gray-800 to-gray-900 flex justify-center items-center">
+        <ClipLoader color={'#0EA5E9'} loading={true} size={150}/>
+      </div>
+    );
+  }
+
+  return (
+    <div className="h-screen overflow-hidden bg-gradient-to-b from-gray-900 via-gray-800 to-gray-900">
+      <button 
+        className="fixed top-4 right-4 z-10 p-2 text-gray-400 hover:text-cyan-400 transition-colors duration-200"
+        onClick={() => setSettingsOpen(true)}
+        aria-label="Settings"
+      >
+        ⚙️
+      </button>
+      
+      {userName && userName.length > 0 && (
+        <Chat 
+          userId={userId} 
+          userName={userName} 
+          aiName={aiName}
+          language={language}
+          aiMemorySelfModify={aiMemorySelfModify}
+          aiStyle={aiStyle}
+          voice={voice}
+        />
+      )}
+      <SettingsModal 
+        aiName={aiName}
+        userName={userName}
+        userId={userId}
+        voice={voice}
+        aiStyle={aiStyle}
+        language={language}
+        isOpen={settingsOpen}
+        onClose={onCloseSettings}
+        onSave={onSaveSettings}
+      />
+    </div>
+  );
+}
+
+export default App;

package/helper-apps/cortex-realtime-voice-server/client/src/SettingsModal.tsx

@@ -0,0 +1,207 @@
+import {ChangeEvent, FormEvent, useState} from 'react';
+import type { Voice, OpenAIVoice, AzureVoice } from '../../src/realtime/realtimeTypes';
+
+type SettingsModalProps = {
+  aiName: string;
+  userName: string;
+  userId: string;
+  voice: Voice;
+  aiStyle: string;
+  language: string;
+  isOpen: boolean;
+  onClose: () => void;
+  onSave: (settings: SettingsData) => void;
+}
+
+export type SettingsData = {
+  userName: string;
+  userId: string;
+  aiName: string;
+  language: string;
+  aiMemorySelfModify: boolean;
+  aiStyle: string;
+  voice: Voice;
+}
+
+// Define voice lists
+const openaiVoices: OpenAIVoice[] = ['alloy', 'echo', 'shimmer', 'ash', 'ballad', 'coral', 'sage', 'verse'];
+const azureVoices: AzureVoice[] = ['amuch', 'dan', 'elan', 'marilyn', 'meadow', 'breeze', 'cove', 'ember', 'jupiter', 'alloy', 'echo', 'shimmer'];
+
+// Check if we're using Azure based on the environment variable
+const isAzure = import.meta.env.VITE_VOICE_PROVIDER === 'azure';
+
+export const SettingsModal = (
+  {aiName, userName, userId, voice, aiStyle, language, isOpen, onClose, onSave}: SettingsModalProps
+) => {
+  const [formData, setFormData] = useState<SettingsData>({
+    aiName,
+    userName,
+    userId,
+    language,
+    aiMemorySelfModify: false,
+    aiStyle,
+    voice
+  });
+
+  const handleChange = (e: ChangeEvent<HTMLInputElement | HTMLSelectElement>) => {
+    const { name, value, type } = e.target;
+    setFormData({
+      ...formData,
+      [name]: type === 'checkbox' ? (e.target as HTMLInputElement).checked : value
+    });
+  };
+
+  const handleSubmit = (e: FormEvent) => {
+    e.preventDefault();
+    onSave(formData);
+    onClose();
+  };
+
+  return (
+    <div className={`fixed inset-0 z-50 overflow-y-auto ${isOpen ? 'block' : 'hidden'}`}>
+      <div className="flex items-center justify-center min-h-screen px-4 pt-4 pb-20 text-center sm:block sm:p-0">
+        <div className="fixed inset-0 transition-opacity bg-gray-900 bg-opacity-75 backdrop-blur-sm" aria-hidden="true" />
+
+        <span className="hidden sm:inline-block sm:align-middle sm:h-screen" aria-hidden="true">&#8203;</span>
+
+        <div className="inline-block w-full max-w-md p-6 my-8 overflow-hidden text-left align-middle transition-all transform bg-gray-800 border border-gray-700/50 rounded-2xl shadow-2xl sm:align-middle">
+          <h3 className="text-lg font-medium leading-6 text-gray-100 mb-4">
+            System Configuration
+          </h3>
+          
+          <form onSubmit={handleSubmit} className="space-y-4">
+            <div>
+              <label className="block text-sm font-medium text-gray-300 mb-1">
+                Your Name
+              </label>
+              <input
+                type="text"
+                name="userName"
+                className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
+                value={formData.userName}
+                onChange={handleChange}
+              />
+            </div>
+
+            <div>
+              <label className="block text-sm font-medium text-gray-300 mb-1">
+                AI Name
+              </label>
+              <input
+                type="text"
+                name="aiName"
+                className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
+                value={formData.aiName}
+                onChange={handleChange}
+              />
+            </div>
+
+            <div>
+              <label className="block text-sm font-medium text-gray-300 mb-1">
+                Voice
+              </label>
+              <select
+                name="voice"
+                className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
+                value={formData.voice}
+                onChange={handleChange}
+              >
+                {isAzure ? (
+                  <optgroup label="Azure Voices">
+                    {azureVoices.map(voice => (
+                      <option key={voice} value={voice}>
+                        {voice.charAt(0).toUpperCase() + voice.slice(1)}
+                      </option>
+                    ))}
+                  </optgroup>
+                ) : (
+                  <optgroup label="OpenAI Voices">
+                    {openaiVoices.map(voice => (
+                      <option key={voice} value={voice}>
+                        {voice.charAt(0).toUpperCase() + voice.slice(1)}
+                      </option>
+                    ))}
+                  </optgroup>
+                )}
+              </select>
+            </div>
+
+            <div>
+              <label className="block text-sm font-medium text-gray-300 mb-1">
+                Language
+              </label>
+              <select
+                name="language"
+                className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
+                value={formData.language}
+                onChange={handleChange}
+              >
+                <option value="en">English</option>
+                <option value="es">Español</option>
+                <option value="fr">Français</option>
+                <option value="de">Deutsch</option>
+                <option value="it">Italiano</option>
+                <option value="pt">Português</option>
+                <option value="ru">Русский</option>
+                <option value="zh">中文</option>
+                <option value="ja">日本語</option>
+                <option value="ko">한국어</option>
+              </select>
+            </div>
+
+            <div>
+              <label className="block text-sm font-medium text-gray-300 mb-1">
+                AI Style
+              </label>
+              <select
+                name="aiStyle"
+                className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
+                value={formData.aiStyle}
+                onChange={handleChange}
+              >
+                <option value="Anthropic">Anthropic</option>
+                <option value="OpenAI">OpenAI</option>
+              </select>
+            </div>
+
+            <div>
+              <label className="block text-sm font-medium text-gray-300 mb-1">
+                Memory Key
+              </label>
+              <input
+                type="text"
+                name="userId"
+                className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
+                value={formData.userId}
+                onChange={handleChange}
+              />
+            </div>
+
+            <div className="flex items-center">
+              <input
+                type="checkbox"
+                id="aiMemorySelfModify"
+                name="aiMemorySelfModify"
+                checked={formData.aiMemorySelfModify}
+                onChange={handleChange}
+                className="w-4 h-4 border-gray-600 rounded bg-gray-700 text-cyan-500 focus:ring-cyan-500/50"
+              />
+              <label htmlFor="aiMemorySelfModify" className="ml-2 text-sm text-gray-300">
+                Enable AI Memory Self-Modification
+              </label>
+            </div>
+
+            <div className="flex justify-end mt-6">
+              <button
+                type="submit"
+                className="px-4 py-2 bg-gradient-to-r from-blue-500 to-cyan-500 hover:from-blue-600 hover:to-cyan-600 text-white rounded-lg shadow-lg shadow-cyan-500/20 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
+              >
+                Save Configuration
+              </button>
+            </div>
+          </form>
+        </div>
+      </div>
+    </div>
+  );
+}