@aj-archipelago/cortex 1.3.4 → 1.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/helper-apps/cortex-autogen/agents.py +90 -20
  2. package/helper-apps/cortex-realtime-voice-server/.env.sample +6 -0
  3. package/helper-apps/cortex-realtime-voice-server/README.md +22 -0
  4. package/helper-apps/cortex-realtime-voice-server/bun.lockb +0 -0
  5. package/helper-apps/cortex-realtime-voice-server/client/bun.lockb +0 -0
  6. package/helper-apps/cortex-realtime-voice-server/client/index.html +12 -0
  7. package/helper-apps/cortex-realtime-voice-server/client/package.json +65 -0
  8. package/helper-apps/cortex-realtime-voice-server/client/postcss.config.js +6 -0
  9. package/helper-apps/cortex-realtime-voice-server/client/public/favicon.ico +0 -0
  10. package/helper-apps/cortex-realtime-voice-server/client/public/index.html +43 -0
  11. package/helper-apps/cortex-realtime-voice-server/client/public/logo192.png +0 -0
  12. package/helper-apps/cortex-realtime-voice-server/client/public/logo512.png +0 -0
  13. package/helper-apps/cortex-realtime-voice-server/client/public/manifest.json +25 -0
  14. package/helper-apps/cortex-realtime-voice-server/client/public/robots.txt +3 -0
  15. package/helper-apps/cortex-realtime-voice-server/client/public/sounds/connect.mp3 +0 -0
  16. package/helper-apps/cortex-realtime-voice-server/client/public/sounds/disconnect.mp3 +0 -0
  17. package/helper-apps/cortex-realtime-voice-server/client/src/App.test.tsx +9 -0
  18. package/helper-apps/cortex-realtime-voice-server/client/src/App.tsx +126 -0
  19. package/helper-apps/cortex-realtime-voice-server/client/src/SettingsModal.tsx +207 -0
  20. package/helper-apps/cortex-realtime-voice-server/client/src/chat/Chat.tsx +553 -0
  21. package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatBubble.tsx +22 -0
  22. package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatBubbleLeft.tsx +22 -0
  23. package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatBubbleRight.tsx +21 -0
  24. package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatMessage.tsx +27 -0
  25. package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatMessageInput.tsx +74 -0
  26. package/helper-apps/cortex-realtime-voice-server/client/src/chat/ChatTile.tsx +211 -0
  27. package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/SoundEffects.ts +56 -0
  28. package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/WavPacker.ts +112 -0
  29. package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/WavRecorder.ts +571 -0
  30. package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/WavStreamPlayer.ts +290 -0
  31. package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/analysis/AudioAnalysis.ts +186 -0
  32. package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/analysis/constants.ts +59 -0
  33. package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/worklets/AudioProcessor.ts +214 -0
  34. package/helper-apps/cortex-realtime-voice-server/client/src/chat/audio/worklets/StreamProcessor.ts +183 -0
  35. package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/AudioVisualizer.tsx +151 -0
  36. package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/CopyButton.tsx +32 -0
  37. package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/ImageOverlay.tsx +166 -0
  38. package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/MicrophoneVisualizer.tsx +95 -0
  39. package/helper-apps/cortex-realtime-voice-server/client/src/chat/components/ScreenshotCapture.tsx +116 -0
  40. package/helper-apps/cortex-realtime-voice-server/client/src/chat/hooks/useWindowResize.ts +27 -0
  41. package/helper-apps/cortex-realtime-voice-server/client/src/chat/utils/audio.ts +33 -0
  42. package/helper-apps/cortex-realtime-voice-server/client/src/index.css +20 -0
  43. package/helper-apps/cortex-realtime-voice-server/client/src/index.tsx +19 -0
  44. package/helper-apps/cortex-realtime-voice-server/client/src/logo.svg +1 -0
  45. package/helper-apps/cortex-realtime-voice-server/client/src/react-app-env.d.ts +1 -0
  46. package/helper-apps/cortex-realtime-voice-server/client/src/reportWebVitals.ts +15 -0
  47. package/helper-apps/cortex-realtime-voice-server/client/src/setupTests.ts +5 -0
  48. package/helper-apps/cortex-realtime-voice-server/client/src/utils/logger.ts +45 -0
  49. package/helper-apps/cortex-realtime-voice-server/client/tailwind.config.js +14 -0
  50. package/helper-apps/cortex-realtime-voice-server/client/tsconfig.json +30 -0
  51. package/helper-apps/cortex-realtime-voice-server/client/vite.config.ts +22 -0
  52. package/helper-apps/cortex-realtime-voice-server/index.ts +19 -0
  53. package/helper-apps/cortex-realtime-voice-server/package.json +28 -0
  54. package/helper-apps/cortex-realtime-voice-server/src/ApiServer.ts +35 -0
  55. package/helper-apps/cortex-realtime-voice-server/src/SocketServer.ts +769 -0
  56. package/helper-apps/cortex-realtime-voice-server/src/Tools.ts +546 -0
  57. package/helper-apps/cortex-realtime-voice-server/src/cortex/expert.ts +29 -0
  58. package/helper-apps/cortex-realtime-voice-server/src/cortex/image.ts +29 -0
  59. package/helper-apps/cortex-realtime-voice-server/src/cortex/memory.ts +89 -0
  60. package/helper-apps/cortex-realtime-voice-server/src/cortex/reason.ts +29 -0
  61. package/helper-apps/cortex-realtime-voice-server/src/cortex/search.ts +30 -0
  62. package/helper-apps/cortex-realtime-voice-server/src/cortex/style.ts +31 -0
  63. package/helper-apps/cortex-realtime-voice-server/src/cortex/utils.ts +94 -0
  64. package/helper-apps/cortex-realtime-voice-server/src/cortex/vision.ts +34 -0
  65. package/helper-apps/cortex-realtime-voice-server/src/realtime/client.ts +484 -0
  66. package/helper-apps/cortex-realtime-voice-server/src/realtime/realtimeTypes.ts +279 -0
  67. package/helper-apps/cortex-realtime-voice-server/src/realtime/socket.ts +27 -0
  68. package/helper-apps/cortex-realtime-voice-server/src/realtime/transcription.ts +75 -0
  69. package/helper-apps/cortex-realtime-voice-server/src/realtime/utils.ts +33 -0
  70. package/helper-apps/cortex-realtime-voice-server/src/utils/logger.ts +45 -0
  71. package/helper-apps/cortex-realtime-voice-server/src/utils/prompt.ts +81 -0
  72. package/helper-apps/cortex-realtime-voice-server/tsconfig.json +28 -0
  73. package/package.json +1 -1
  74. package/pathways/basePathway.js +3 -1
  75. package/pathways/system/entity/memory/sys_memory_manager.js +3 -0
  76. package/pathways/system/entity/memory/sys_memory_update.js +43 -45
  77. package/pathways/system/entity/memory/sys_read_memory.js +86 -6
  78. package/pathways/system/entity/memory/sys_search_memory.js +66 -0
  79. package/pathways/system/entity/shared/sys_entity_constants.js +1 -1
  80. package/pathways/system/entity/sys_entity_continue.js +2 -1
  81. package/pathways/system/entity/sys_entity_start.js +13 -2
  82. package/pathways/system/entity/sys_generator_ack.js +2 -2
  83. package/pathways/system/entity/sys_generator_expert.js +0 -2
  84. package/pathways/system/entity/sys_generator_memory.js +31 -0
  85. package/pathways/system/entity/sys_generator_quick.js +22 -7
  86. package/pathways/system/entity/sys_generator_reasoning.js +1 -1
  87. package/pathways/system/entity/sys_generator_results.js +20 -16
  88. package/pathways/system/entity/sys_generator_voice_filler.js +1 -1
  89. package/pathways/system/entity/sys_generator_voice_sample.js +36 -0
  90. package/pathways/system/entity/sys_router_tool.js +13 -10
  91. package/pathways/system/sys_parse_numbered_object_list.js +1 -1
  92. package/server/pathwayResolver.js +41 -31
  93. package/server/plugins/azureVideoTranslatePlugin.js +28 -16
  94. package/server/plugins/claude3VertexPlugin.js +0 -9
  95. package/server/plugins/gemini15ChatPlugin.js +18 -5
  96. package/server/plugins/modelPlugin.js +27 -6
  97. package/server/plugins/openAiChatPlugin.js +10 -8
  98. package/server/plugins/openAiVisionPlugin.js +56 -0
  99. package/tests/memoryfunction.test.js +73 -1
@@ -0,0 +1,207 @@
1
+ import {ChangeEvent, FormEvent, useState} from 'react';
2
+ import type { Voice, OpenAIVoice, AzureVoice } from '../../src/realtime/realtimeTypes';
3
+
4
+ type SettingsModalProps = {
5
+ aiName: string;
6
+ userName: string;
7
+ userId: string;
8
+ voice: Voice;
9
+ aiStyle: string;
10
+ language: string;
11
+ isOpen: boolean;
12
+ onClose: () => void;
13
+ onSave: (settings: SettingsData) => void;
14
+ }
15
+
16
+ export type SettingsData = {
17
+ userName: string;
18
+ userId: string;
19
+ aiName: string;
20
+ language: string;
21
+ aiMemorySelfModify: boolean;
22
+ aiStyle: string;
23
+ voice: Voice;
24
+ }
25
+
26
+ // Define voice lists
27
+ const openaiVoices: OpenAIVoice[] = ['alloy', 'echo', 'shimmer', 'ash', 'ballad', 'coral', 'sage', 'verse'];
28
+ const azureVoices: AzureVoice[] = ['amuch', 'dan', 'elan', 'marilyn', 'meadow', 'breeze', 'cove', 'ember', 'jupiter', 'alloy', 'echo', 'shimmer'];
29
+
30
+ // Check if we're using Azure based on the environment variable
31
+ const isAzure = import.meta.env.VITE_VOICE_PROVIDER === 'azure';
32
+
33
+ export const SettingsModal = (
34
+ {aiName, userName, userId, voice, aiStyle, language, isOpen, onClose, onSave}: SettingsModalProps
35
+ ) => {
36
+ const [formData, setFormData] = useState<SettingsData>({
37
+ aiName,
38
+ userName,
39
+ userId,
40
+ language,
41
+ aiMemorySelfModify: false,
42
+ aiStyle,
43
+ voice
44
+ });
45
+
46
+ const handleChange = (e: ChangeEvent<HTMLInputElement | HTMLSelectElement>) => {
47
+ const { name, value, type } = e.target;
48
+ setFormData({
49
+ ...formData,
50
+ [name]: type === 'checkbox' ? (e.target as HTMLInputElement).checked : value
51
+ });
52
+ };
53
+
54
+ const handleSubmit = (e: FormEvent) => {
55
+ e.preventDefault();
56
+ onSave(formData);
57
+ onClose();
58
+ };
59
+
60
+ return (
61
+ <div className={`fixed inset-0 z-50 overflow-y-auto ${isOpen ? 'block' : 'hidden'}`}>
62
+ <div className="flex items-center justify-center min-h-screen px-4 pt-4 pb-20 text-center sm:block sm:p-0">
63
+ <div className="fixed inset-0 transition-opacity bg-gray-900 bg-opacity-75 backdrop-blur-sm" aria-hidden="true" />
64
+
65
+ <span className="hidden sm:inline-block sm:align-middle sm:h-screen" aria-hidden="true">&#8203;</span>
66
+
67
+ <div className="inline-block w-full max-w-md p-6 my-8 overflow-hidden text-left align-middle transition-all transform bg-gray-800 border border-gray-700/50 rounded-2xl shadow-2xl sm:align-middle">
68
+ <h3 className="text-lg font-medium leading-6 text-gray-100 mb-4">
69
+ System Configuration
70
+ </h3>
71
+
72
+ <form onSubmit={handleSubmit} className="space-y-4">
73
+ <div>
74
+ <label className="block text-sm font-medium text-gray-300 mb-1">
75
+ Your Name
76
+ </label>
77
+ <input
78
+ type="text"
79
+ name="userName"
80
+ className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
81
+ value={formData.userName}
82
+ onChange={handleChange}
83
+ />
84
+ </div>
85
+
86
+ <div>
87
+ <label className="block text-sm font-medium text-gray-300 mb-1">
88
+ AI Name
89
+ </label>
90
+ <input
91
+ type="text"
92
+ name="aiName"
93
+ className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
94
+ value={formData.aiName}
95
+ onChange={handleChange}
96
+ />
97
+ </div>
98
+
99
+ <div>
100
+ <label className="block text-sm font-medium text-gray-300 mb-1">
101
+ Voice
102
+ </label>
103
+ <select
104
+ name="voice"
105
+ className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
106
+ value={formData.voice}
107
+ onChange={handleChange}
108
+ >
109
+ {isAzure ? (
110
+ <optgroup label="Azure Voices">
111
+ {azureVoices.map(voice => (
112
+ <option key={voice} value={voice}>
113
+ {voice.charAt(0).toUpperCase() + voice.slice(1)}
114
+ </option>
115
+ ))}
116
+ </optgroup>
117
+ ) : (
118
+ <optgroup label="OpenAI Voices">
119
+ {openaiVoices.map(voice => (
120
+ <option key={voice} value={voice}>
121
+ {voice.charAt(0).toUpperCase() + voice.slice(1)}
122
+ </option>
123
+ ))}
124
+ </optgroup>
125
+ )}
126
+ </select>
127
+ </div>
128
+
129
+ <div>
130
+ <label className="block text-sm font-medium text-gray-300 mb-1">
131
+ Language
132
+ </label>
133
+ <select
134
+ name="language"
135
+ className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
136
+ value={formData.language}
137
+ onChange={handleChange}
138
+ >
139
+ <option value="en">English</option>
140
+ <option value="es">Español</option>
141
+ <option value="fr">Français</option>
142
+ <option value="de">Deutsch</option>
143
+ <option value="it">Italiano</option>
144
+ <option value="pt">Português</option>
145
+ <option value="ru">Русский</option>
146
+ <option value="zh">中文</option>
147
+ <option value="ja">日本語</option>
148
+ <option value="ko">한국어</option>
149
+ </select>
150
+ </div>
151
+
152
+ <div>
153
+ <label className="block text-sm font-medium text-gray-300 mb-1">
154
+ AI Style
155
+ </label>
156
+ <select
157
+ name="aiStyle"
158
+ className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
159
+ value={formData.aiStyle}
160
+ onChange={handleChange}
161
+ >
162
+ <option value="Anthropic">Anthropic</option>
163
+ <option value="OpenAI">OpenAI</option>
164
+ </select>
165
+ </div>
166
+
167
+ <div>
168
+ <label className="block text-sm font-medium text-gray-300 mb-1">
169
+ Memory Key
170
+ </label>
171
+ <input
172
+ type="text"
173
+ name="userId"
174
+ className="w-full px-3 py-2 bg-gray-700/50 border border-gray-600/50 rounded-lg text-gray-100 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
175
+ value={formData.userId}
176
+ onChange={handleChange}
177
+ />
178
+ </div>
179
+
180
+ <div className="flex items-center">
181
+ <input
182
+ type="checkbox"
183
+ id="aiMemorySelfModify"
184
+ name="aiMemorySelfModify"
185
+ checked={formData.aiMemorySelfModify}
186
+ onChange={handleChange}
187
+ className="w-4 h-4 border-gray-600 rounded bg-gray-700 text-cyan-500 focus:ring-cyan-500/50"
188
+ />
189
+ <label htmlFor="aiMemorySelfModify" className="ml-2 text-sm text-gray-300">
190
+ Enable AI Memory Self-Modification
191
+ </label>
192
+ </div>
193
+
194
+ <div className="flex justify-end mt-6">
195
+ <button
196
+ type="submit"
197
+ className="px-4 py-2 bg-gradient-to-r from-blue-500 to-cyan-500 hover:from-blue-600 hover:to-cyan-600 text-white rounded-lg shadow-lg shadow-cyan-500/20 focus:outline-none focus:ring-2 focus:ring-cyan-500/50"
198
+ >
199
+ Save Configuration
200
+ </button>
201
+ </div>
202
+ </form>
203
+ </div>
204
+ </div>
205
+ </div>
206
+ );
207
+ }