@aj-archipelago/cortex 1.3.4 → 1.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/helper-apps/cortex-autogen/agents.py +90 -20
- package/helper-apps/cortex-realtime-voice-server/.env.sample +6 -0
- package/helper-apps/cortex-realtime-voice-server/README.md +22 -0
- package/helper-apps/cortex-realtime-voice-server/bun.lockb +0 -0
- package/helper-apps/cortex-realtime-voice-server/client/bun.lockb +0 -0
- package/helper-apps/cortex-realtime-voice-server/client/index.html +12 -0
- package/helper-apps/cortex-realtime-voice-server/client/package.json +65 -0
- package/helper-apps/cortex-realtime-voice-server/client/postcss.config.js +6 -0
- package/helper-apps/cortex-realtime-voice-server/client/public/favicon.ico +0 -0
- package/helper-apps/cortex-realtime-voice-server/client/public/index.html +43 -0
- package/helper-apps/cortex-realtime-voice-server/client/public/logo192.png +0 -0
- package/helper-apps/cortex-realtime-voice-server/client/public/logo512.png +0 -0
- package/helper-apps/cortex-realtime-voice-server/client/public/manifest.json +25 -0
- package/helper-apps/cortex-realtime-voice-server/client/public/robots.txt +3 -0
- package/helper-apps/cortex-realtime-voice-server/client/public/sounds/connect.mp3 +0 -0
- package/helper-apps/cortex-realtime-voice-server/client/public/sounds/disconnect.mp3 +0 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/App.test.tsx +9 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/App.tsx +126 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/SettingsModal.tsx +207 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/Chat.tsx +553 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatBubble.tsx +22 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatBubbleLeft.tsx +22 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatBubbleRight.tsx +21 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatMessage.tsx +27 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatMessageInput.tsx +74 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatTile.tsx +211 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/SoundEffects.ts +56 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/WavPacker.ts +112 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/WavRecorder.ts +571 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/WavStreamPlayer.ts +290 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/analysis/AudioAnalysis.ts +186 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/analysis/constants.ts +59 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/worklets/AudioProcessor.ts +214 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/worklets/StreamProcessor.ts +183 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/AudioVisualizer.tsx +151 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/CopyButton.tsx +32 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/ImageOverlay.tsx +166 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/MicrophoneVisualizer.tsx +95 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/ScreenshotCapture.tsx +116 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/hooks/useWindowResize.ts +27 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/utils/audio.ts +33 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/index.css +20 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/index.tsx +19 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/logo.svg +1 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/react-app-env.d.ts +1 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/reportWebVitals.ts +15 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/setupTests.ts +5 -0
- package/helper-apps/cortex-realtime-voice-server/client/src/utils/logger.ts +45 -0
- package/helper-apps/cortex-realtime-voice-server/client/tailwind.config.js +14 -0
- package/helper-apps/cortex-realtime-voice-server/client/tsconfig.json +30 -0
- package/helper-apps/cortex-realtime-voice-server/client/vite.config.ts +22 -0
- package/helper-apps/cortex-realtime-voice-server/index.ts +19 -0
- package/helper-apps/cortex-realtime-voice-server/package.json +28 -0
- package/helper-apps/cortex-realtime-voice-server/src/ApiServer.ts +35 -0
- package/helper-apps/cortex-realtime-voice-server/src/SocketServer.ts +769 -0
- package/helper-apps/cortex-realtime-voice-server/src/Tools.ts +546 -0
- package/helper-apps/cortex-realtime-voice-server/src/cortex/expert.ts +29 -0
- package/helper-apps/cortex-realtime-voice-server/src/cortex/image.ts +29 -0
- package/helper-apps/cortex-realtime-voice-server/src/cortex/memory.ts +89 -0
- package/helper-apps/cortex-realtime-voice-server/src/cortex/reason.ts +29 -0
- package/helper-apps/cortex-realtime-voice-server/src/cortex/search.ts +30 -0
- package/helper-apps/cortex-realtime-voice-server/src/cortex/style.ts +31 -0
- package/helper-apps/cortex-realtime-voice-server/src/cortex/utils.ts +94 -0
- package/helper-apps/cortex-realtime-voice-server/src/cortex/vision.ts +34 -0
- package/helper-apps/cortex-realtime-voice-server/src/realtime/client.ts +484 -0
- package/helper-apps/cortex-realtime-voice-server/src/realtime/realtimeTypes.ts +279 -0
- package/helper-apps/cortex-realtime-voice-server/src/realtime/socket.ts +27 -0
- package/helper-apps/cortex-realtime-voice-server/src/realtime/transcription.ts +75 -0
- package/helper-apps/cortex-realtime-voice-server/src/realtime/utils.ts +33 -0
- package/helper-apps/cortex-realtime-voice-server/src/utils/logger.ts +45 -0
- package/helper-apps/cortex-realtime-voice-server/src/utils/prompt.ts +81 -0
- package/helper-apps/cortex-realtime-voice-server/tsconfig.json +28 -0
- package/package.json +1 -1
- package/pathways/basePathway.js +3 -1
- package/pathways/system/entity/memory/sys_memory_manager.js +3 -0
- package/pathways/system/entity/memory/sys_memory_update.js +43 -45
- package/pathways/system/entity/memory/sys_read_memory.js +86 -6
- package/pathways/system/entity/memory/sys_search_memory.js +66 -0
- package/pathways/system/entity/shared/sys_entity_constants.js +1 -1
- package/pathways/system/entity/sys_entity_continue.js +2 -1
- package/pathways/system/entity/sys_entity_start.js +13 -2
- package/pathways/system/entity/sys_generator_ack.js +2 -2
- package/pathways/system/entity/sys_generator_expert.js +0 -2
- package/pathways/system/entity/sys_generator_memory.js +31 -0
- package/pathways/system/entity/sys_generator_quick.js +22 -7
- package/pathways/system/entity/sys_generator_reasoning.js +1 -1
- package/pathways/system/entity/sys_generator_results.js +20 -16
- package/pathways/system/entity/sys_generator_voice_filler.js +1 -1
- package/pathways/system/entity/sys_generator_voice_sample.js +36 -0
- package/pathways/system/entity/sys_router_tool.js +13 -10
- package/pathways/system/sys_parse_numbered_object_list.js +1 -1
- package/server/pathwayResolver.js +41 -31
- package/server/plugins/azureVideoTranslatePlugin.js +28 -16
- package/server/plugins/claude3VertexPlugin.js +0 -9
- package/server/plugins/gemini15ChatPlugin.js +18 -5
- package/server/plugins/modelPlugin.js +27 -6
- package/server/plugins/openAiChatPlugin.js +10 -8
- package/server/plugins/openAiVisionPlugin.js +56 -0
- package/tests/memoryfunction.test.js +73 -1
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import {ChangeEvent, FormEvent, useState} from 'react';
|
|
2
|
+
import type { Voice, OpenAIVoice, AzureVoice } from '../../src/realtime/realtimeTypes';
|
|
3
|
+
|
|
4
|
+
type SettingsModalProps = {
|
|
5
|
+
aiName: string;
|
|
6
|
+
userName: string;
|
|
7
|
+
userId: string;
|
|
8
|
+
voice: Voice;
|
|
9
|
+
aiStyle: string;
|
|
10
|
+
language: string;
|
|
11
|
+
isOpen: boolean;
|
|
12
|
+
onClose: () => void;
|
|
13
|
+
onSave: (settings: SettingsData) => void;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export type SettingsData = {
|
|
17
|
+
userName: string;
|
|
18
|
+
userId: string;
|
|
19
|
+
aiName: string;
|
|
20
|
+
language: string;
|
|
21
|
+
aiMemorySelfModify: boolean;
|
|
22
|
+
aiStyle: string;
|
|
23
|
+
voice: Voice;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// Define voice lists
|
|
27
|
+
const openaiVoices: OpenAIVoice[] = ['alloy', 'echo', 'shimmer', 'ash', 'ballad', 'coral', 'sage', 'verse'];
|
|
28
|
+
const azureVoices: AzureVoice[] = ['amuch', 'dan', 'elan', 'marilyn', 'meadow', 'breeze', 'cove', 'ember', 'jupiter', 'alloy', 'echo', 'shimmer'];
|
|
29
|
+
|
|
30
|
+
// Check if we're using Azure based on the environment variable
|
|
31
|
+
const isAzure = import.meta.env.VITE_VOICE_PROVIDER === 'azure';
|
|
32
|
+
|
|
33
|
+
export const SettingsModal = (
|
|
34
|
+
{aiName, userName, userId, voice, aiStyle, language, isOpen, onClose, onSave}: SettingsModalProps
|
|
35
|
+
) => {
|
|
36
|
+
const [formData, setFormData] = useState<SettingsData>({
|
|
37
|
+
aiName,
|
|
38
|
+
userName,
|
|
39
|
+
userId,
|
|
40
|
+
language,
|
|
41
|
+
aiMemorySelfModify: false,
|
|
42
|
+
aiStyle,
|
|
43
|
+
voice
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
const handleChange = (e: ChangeEvent<HTMLInputElement | HTMLSelectElement>) => {
|
|
47
|
+
const { name, value, type } = e.target;
|
|
48
|
+
setFormData({
|
|
49
|
+
...formData,
|
|
50
|
+
[name]: type === 'checkbox' ? (e.target as HTMLInputElement).checked : value
|
|
51
|
+
});
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
const handleSubmit = (e: FormEvent) => {
|
|
55
|
+
e.preventDefault();
|
|
56
|
+
onSave(formData);
|
|
57
|
+
onClose();
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
return (
|
|
61
|
+
<div className={`fixed inset-0 z-50 overflow-y-auto ${isOpen ? 'block' : 'hidden'}`}>
|
|
62
|
+
<div className="flex items-center justify-center min-h-screen px-4 pt-4 pb-20 text-center sm:block sm:p-0">
|
|
63
|
+
<div className="fixed inset-0 transition-opacity bg-gray-900 bg-opacity-75 backdrop-blur-sm" aria-hidden="true" />
|
|
64
|
+
|
|
65
|
+
<span className="hidden sm:inline-block sm:align-middle sm:h-screen" aria-hidden="true">​</span>
|
|
66
|
+
|
|
67
|
+
<div className="inline-block w-full max-w-md p-6 my-8 overflow-hidden text-left align-middle transition-all transform bg-gray-800 border border-gray-700/50 rounded-2xl shadow-2xl sm:align-middle">
|
|
68
|
+
<h3 className="text-lg font-medium leading-6 text-gray-100 mb-4">
|
|
69
|
+
System Configuration
|
|
70
|
+
</h3>
|
|
71
|
+
|
|
72
|
+
<form onSubmit={handleSubmit} className="space-y-4">
|
|
73
|
+
<div>
|
|
74
|
+
<label className="block text-sm font-medium text-gray-300 mb-1">
|
|
75
|
+
Your Name
|
|
76
|
+
</label>
|
|
77
|
+
<input
|
|
78
|
+
type="text"
|
|
79
|
+
name="userName"
|
|
80
|
+
className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
|
|
81
|
+
value={formData.userName}
|
|
82
|
+
onChange={handleChange}
|
|
83
|
+
/>
|
|
84
|
+
</div>
|
|
85
|
+
|
|
86
|
+
<div>
|
|
87
|
+
<label className="block text-sm font-medium text-gray-300 mb-1">
|
|
88
|
+
AI Name
|
|
89
|
+
</label>
|
|
90
|
+
<input
|
|
91
|
+
type="text"
|
|
92
|
+
name="aiName"
|
|
93
|
+
className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
|
|
94
|
+
value={formData.aiName}
|
|
95
|
+
onChange={handleChange}
|
|
96
|
+
/>
|
|
97
|
+
</div>
|
|
98
|
+
|
|
99
|
+
<div>
|
|
100
|
+
<label className="block text-sm font-medium text-gray-300 mb-1">
|
|
101
|
+
Voice
|
|
102
|
+
</label>
|
|
103
|
+
<select
|
|
104
|
+
name="voice"
|
|
105
|
+
className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
|
|
106
|
+
value={formData.voice}
|
|
107
|
+
onChange={handleChange}
|
|
108
|
+
>
|
|
109
|
+
{isAzure ? (
|
|
110
|
+
<optgroup label="Azure Voices">
|
|
111
|
+
{azureVoices.map(voice => (
|
|
112
|
+
<option key={voice} value={voice}>
|
|
113
|
+
{voice.charAt(0).toUpperCase() + voice.slice(1)}
|
|
114
|
+
</option>
|
|
115
|
+
))}
|
|
116
|
+
</optgroup>
|
|
117
|
+
) : (
|
|
118
|
+
<optgroup label="OpenAI Voices">
|
|
119
|
+
{openaiVoices.map(voice => (
|
|
120
|
+
<option key={voice} value={voice}>
|
|
121
|
+
{voice.charAt(0).toUpperCase() + voice.slice(1)}
|
|
122
|
+
</option>
|
|
123
|
+
))}
|
|
124
|
+
</optgroup>
|
|
125
|
+
)}
|
|
126
|
+
</select>
|
|
127
|
+
</div>
|
|
128
|
+
|
|
129
|
+
<div>
|
|
130
|
+
<label className="block text-sm font-medium text-gray-300 mb-1">
|
|
131
|
+
Language
|
|
132
|
+
</label>
|
|
133
|
+
<select
|
|
134
|
+
name="language"
|
|
135
|
+
className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
|
|
136
|
+
value={formData.language}
|
|
137
|
+
onChange={handleChange}
|
|
138
|
+
>
|
|
139
|
+
<option value="en">English</option>
|
|
140
|
+
<option value="es">Español</option>
|
|
141
|
+
<option value="fr">Français</option>
|
|
142
|
+
<option value="de">Deutsch</option>
|
|
143
|
+
<option value="it">Italiano</option>
|
|
144
|
+
<option value="pt">Português</option>
|
|
145
|
+
<option value="ru">Русский</option>
|
|
146
|
+
<option value="zh">中文</option>
|
|
147
|
+
<option value="ja">日本語</option>
|
|
148
|
+
<option value="ko">한국어</option>
|
|
149
|
+
</select>
|
|
150
|
+
</div>
|
|
151
|
+
|
|
152
|
+
<div>
|
|
153
|
+
<label className="block text-sm font-medium text-gray-300 mb-1">
|
|
154
|
+
AI Style
|
|
155
|
+
</label>
|
|
156
|
+
<select
|
|
157
|
+
name="aiStyle"
|
|
158
|
+
className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
|
|
159
|
+
value={formData.aiStyle}
|
|
160
|
+
onChange={handleChange}
|
|
161
|
+
>
|
|
162
|
+
<option value="Anthropic">Anthropic</option>
|
|
163
|
+
<option value="OpenAI">OpenAI</option>
|
|
164
|
+
</select>
|
|
165
|
+
</div>
|
|
166
|
+
|
|
167
|
+
<div>
|
|
168
|
+
<label className="block text-sm font-medium text-gray-300 mb-1">
|
|
169
|
+
Memory Key
|
|
170
|
+
</label>
|
|
171
|
+
<input
|
|
172
|
+
type="text"
|
|
173
|
+
name="userId"
|
|
174
|
+
className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
|
|
175
|
+
value={formData.userId}
|
|
176
|
+
onChange={handleChange}
|
|
177
|
+
/>
|
|
178
|
+
</div>
|
|
179
|
+
|
|
180
|
+
<div className="flex items-center">
|
|
181
|
+
<input
|
|
182
|
+
type="checkbox"
|
|
183
|
+
id="aiMemorySelfModify"
|
|
184
|
+
name="aiMemorySelfModify"
|
|
185
|
+
checked={formData.aiMemorySelfModify}
|
|
186
|
+
onChange={handleChange}
|
|
187
|
+
className="w-4 h-4 border-gray-600 rounded bg-gray-700 text-cyan-500 focus:ring-cyan-500/50"
|
|
188
|
+
/>
|
|
189
|
+
<label htmlFor="aiMemorySelfModify" className="ml-2 text-sm text-gray-300">
|
|
190
|
+
Enable AI Memory Self-Modification
|
|
191
|
+
</label>
|
|
192
|
+
</div>
|
|
193
|
+
|
|
194
|
+
<div className="flex justify-end mt-6">
|
|
195
|
+
<button
|
|
196
|
+
type="submit"
|
|
197
|
+
className="px-4 py-2 bg-gradient-to-r from-blue-500 to-cyan-500 hover:from-blue-600 hover:to-cyan-600 text-white rounded-lg shadow-lg shadow-cyan-500/20 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
|
|
198
|
+
>
|
|
199
|
+
Save Configuration
|
|
200
|
+
</button>
|
|
201
|
+
</div>
|
|
202
|
+
</form>
|
|
203
|
+
</div>
|
|
204
|
+
</div>
|
|
205
|
+
</div>
|
|
206
|
+
);
|
|
207
|
+
}
|