@aj-archipelago/cortex 1.3.7 → 1.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +578 -80
- package/helper-apps/cortex-realtime-voice-server/client/src/chat/Chat.tsx +51 -11
- package/helper-apps/cortex-realtime-voice-server/src/SocketServer.ts +220 -183
- package/helper-apps/cortex-realtime-voice-server/src/Tools.ts +18 -34
- package/helper-apps/cortex-realtime-voice-server/src/cortex/utils.ts +29 -15
- package/helper-apps/cortex-realtime-voice-server/src/realtime/client.ts +47 -1
- package/helper-apps/cortex-realtime-voice-server/src/utils/prompt.ts +2 -11
- package/package.json +1 -1
- package/pathways/system/entity/memory/sys_search_memory.js +2 -1
- package/pathways/system/entity/sys_entity_start.js +6 -7
- package/pathways/system/entity/sys_generator_voice_sample.js +2 -2
- package/pathways/translate_gpt4_omni.js +20 -0
- package/pathways/translate_subtitle.js +326 -135
- package/pathways/translate_subtitle_helper.js +4 -16
- package/server/plugins/claude3VertexPlugin.js +10 -17
- package/server/plugins/gemini15VisionPlugin.js +16 -3
- package/server/plugins/modelPlugin.js +27 -0
- package/server/plugins/openAiVisionPlugin.js +26 -8
- package/tests/multimodal_conversion.test.js +88 -12
- package/tests/translate_srt.test.js +66 -14
|
@@ -82,16 +82,35 @@ export default function Chat({
|
|
|
82
82
|
logger.log('Stopping conversation');
|
|
83
83
|
const wavRecorder = wavRecorderRef.current;
|
|
84
84
|
const wavStreamPlayer = wavStreamPlayerRef.current;
|
|
85
|
+
const socket = socketRef.current;
|
|
85
86
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
87
|
+
try {
|
|
88
|
+
// First stop recording and audio playback
|
|
89
|
+
if (wavRecorder.getStatus() === "recording") {
|
|
90
|
+
await wavRecorder.end();
|
|
91
|
+
}
|
|
92
|
+
if (wavStreamPlayer) {
|
|
93
|
+
await wavStreamPlayer.interrupt();
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Clean up socket connection first
|
|
97
|
+
if (socket) {
|
|
98
|
+
// Only emit if we're still connected
|
|
99
|
+
if (socket.connected) {
|
|
100
|
+
socket.emit('conversationCompleted');
|
|
101
|
+
// Wait a bit to ensure the message is sent
|
|
102
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
103
|
+
}
|
|
104
|
+
socket.removeAllListeners();
|
|
105
|
+
socket.disconnect();
|
|
106
|
+
// Create a new socket instance to ensure clean state
|
|
107
|
+
socketRef.current = io(`/?userId=${userId}&userName=${userName}&aiName=${aiName}&voice=${voice}&aiStyle=${aiStyle}&language=${language}`, {
|
|
108
|
+
autoConnect: false,
|
|
109
|
+
reconnection: false
|
|
110
|
+
});
|
|
111
|
+
}
|
|
93
112
|
|
|
94
|
-
//
|
|
113
|
+
// Then clean up audio nodes
|
|
95
114
|
if (sourceNodeRef.current) {
|
|
96
115
|
sourceNodeRef.current.disconnect();
|
|
97
116
|
sourceNodeRef.current = null;
|
|
@@ -109,15 +128,36 @@ export default function Chat({
|
|
|
109
128
|
wavRecorderRef.current = new WavRecorder({sampleRate: 24000});
|
|
110
129
|
wavStreamPlayerRef.current = new WavStreamPlayer({sampleRate: 24000});
|
|
111
130
|
|
|
131
|
+
// Play disconnect sound last
|
|
132
|
+
await SoundEffects.playDisconnect();
|
|
133
|
+
|
|
134
|
+
// Reset state
|
|
135
|
+
setIsRecording(false);
|
|
136
|
+
setIsMuted(false);
|
|
137
|
+
setImageUrls([]);
|
|
138
|
+
setIsAudioPlaying(false);
|
|
139
|
+
} catch (error) {
|
|
140
|
+
logger.error('Error stopping conversation:', error);
|
|
141
|
+
// Even if there's an error, try to reset critical state
|
|
112
142
|
setIsRecording(false);
|
|
113
|
-
|
|
114
|
-
setImageUrls([]); // Clear images on disconnect
|
|
143
|
+
setIsAudioPlaying(false);
|
|
115
144
|
}
|
|
116
|
-
}, []);
|
|
145
|
+
}, [userId, userName, aiName, voice, aiStyle, language]);
|
|
117
146
|
|
|
118
147
|
const startConversation = useCallback(() => {
|
|
119
148
|
logger.log('Starting conversation');
|
|
120
149
|
|
|
150
|
+
// Clean up any existing socket connection first
|
|
151
|
+
if (socketRef.current?.connected) {
|
|
152
|
+
socketRef.current.disconnect();
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// Create a new socket instance
|
|
156
|
+
socketRef.current = io(`/?userId=${userId}&userName=${userName}&aiName=${aiName}&voice=${voice}&aiStyle=${aiStyle}&language=${language}`, {
|
|
157
|
+
autoConnect: false,
|
|
158
|
+
timeout: 10000 // 10 second connection timeout
|
|
159
|
+
});
|
|
160
|
+
|
|
121
161
|
const socket = socketRef.current;
|
|
122
162
|
|
|
123
163
|
// Remove ALL existing listeners before adding new ones
|