openclaw-liveavatar 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/.next/BUILD_ID +1 -0
  2. package/.next/app-build-manifest.json +42 -0
  3. package/.next/app-path-routes-manifest.json +6 -0
  4. package/.next/build-manifest.json +33 -0
  5. package/.next/cache/.previewinfo +1 -0
  6. package/.next/cache/.rscinfo +1 -0
  7. package/.next/cache/.tsbuildinfo +1 -0
  8. package/.next/cache/chrome-devtools-workspace-uuid +1 -0
  9. package/.next/cache/next-devtools-config.json +1 -0
  10. package/.next/cache/webpack/client-production/0.pack +0 -0
  11. package/.next/cache/webpack/client-production/1.pack +0 -0
  12. package/.next/cache/webpack/client-production/2.pack +0 -0
  13. package/.next/cache/webpack/client-production/3.pack +0 -0
  14. package/.next/cache/webpack/client-production/4.pack +0 -0
  15. package/.next/cache/webpack/client-production/index.pack +0 -0
  16. package/.next/cache/webpack/client-production/index.pack.old +0 -0
  17. package/.next/cache/webpack/edge-server-production/0.pack +0 -0
  18. package/.next/cache/webpack/edge-server-production/index.pack +0 -0
  19. package/.next/cache/webpack/server-production/0.pack +0 -0
  20. package/.next/cache/webpack/server-production/index.pack +0 -0
  21. package/.next/diagnostics/build-diagnostics.json +6 -0
  22. package/.next/diagnostics/framework.json +1 -0
  23. package/.next/export-marker.json +6 -0
  24. package/.next/images-manifest.json +58 -0
  25. package/.next/next-minimal-server.js.nft.json +1 -0
  26. package/.next/next-server.js.nft.json +1 -0
  27. package/.next/package.json +1 -0
  28. package/.next/prerender-manifest.json +61 -0
  29. package/.next/react-loadable-manifest.json +1 -0
  30. package/.next/required-server-files.json +320 -0
  31. package/.next/routes-manifest.json +53 -0
  32. package/.next/server/app/_not-found/page.js +5 -0
  33. package/.next/server/app/_not-found/page.js.nft.json +1 -0
  34. package/.next/server/app/_not-found/page_client-reference-manifest.js +1 -0
  35. package/.next/server/app/_not-found.html +4 -0
  36. package/.next/server/app/_not-found.meta +8 -0
  37. package/.next/server/app/_not-found.rsc +15 -0
  38. package/.next/server/app/api/get-avatars/route.js +1 -0
  39. package/.next/server/app/api/get-avatars/route.js.nft.json +1 -0
  40. package/.next/server/app/api/get-avatars/route_client-reference-manifest.js +1 -0
  41. package/.next/server/app/api/start-session/route.js +1 -0
  42. package/.next/server/app/api/start-session/route.js.nft.json +1 -0
  43. package/.next/server/app/api/start-session/route_client-reference-manifest.js +1 -0
  44. package/.next/server/app/index.html +4 -0
  45. package/.next/server/app/index.meta +7 -0
  46. package/.next/server/app/index.rsc +16 -0
  47. package/.next/server/app/page.js +9 -0
  48. package/.next/server/app/page.js.nft.json +1 -0
  49. package/.next/server/app/page_client-reference-manifest.js +1 -0
  50. package/.next/server/app-paths-manifest.json +6 -0
  51. package/.next/server/chunks/361.js +9 -0
  52. package/.next/server/chunks/611.js +6 -0
  53. package/.next/server/chunks/873.js +22 -0
  54. package/.next/server/functions-config-manifest.json +4 -0
  55. package/.next/server/interception-route-rewrite-manifest.js +1 -0
  56. package/.next/server/middleware-build-manifest.js +1 -0
  57. package/.next/server/middleware-manifest.json +6 -0
  58. package/.next/server/middleware-react-loadable-manifest.js +1 -0
  59. package/.next/server/next-font-manifest.js +1 -0
  60. package/.next/server/next-font-manifest.json +1 -0
  61. package/.next/server/pages/404.html +4 -0
  62. package/.next/server/pages/500.html +1 -0
  63. package/.next/server/pages/_app.js +1 -0
  64. package/.next/server/pages/_app.js.nft.json +1 -0
  65. package/.next/server/pages/_document.js +1 -0
  66. package/.next/server/pages/_document.js.nft.json +1 -0
  67. package/.next/server/pages/_error.js +19 -0
  68. package/.next/server/pages/_error.js.nft.json +1 -0
  69. package/.next/server/pages-manifest.json +6 -0
  70. package/.next/server/server-reference-manifest.js +1 -0
  71. package/.next/server/server-reference-manifest.json +1 -0
  72. package/.next/server/webpack-runtime.js +1 -0
  73. package/.next/static/chunks/144d3bae-37bcc55d23f188ee.js +1 -0
  74. package/.next/static/chunks/255-35bf8c00c5dde345.js +1 -0
  75. package/.next/static/chunks/336-a66237a0a1db954a.js +1 -0
  76. package/.next/static/chunks/4bd1b696-c023c6e3521b1417.js +1 -0
  77. package/.next/static/chunks/app/_not-found/page-dfc6e5d8e6c6203c.js +1 -0
  78. package/.next/static/chunks/app/api/get-avatars/route-8017e1cff542d5d0.js +1 -0
  79. package/.next/static/chunks/app/api/start-session/route-8017e1cff542d5d0.js +1 -0
  80. package/.next/static/chunks/app/layout-ff675313cc8f8fcf.js +1 -0
  81. package/.next/static/chunks/app/page-9e4b703722bef650.js +1 -0
  82. package/.next/static/chunks/framework-de98b93a850cfc71.js +1 -0
  83. package/.next/static/chunks/main-1a0dcce460eb61ce.js +1 -0
  84. package/.next/static/chunks/main-app-e7f1007edc7ad7e1.js +1 -0
  85. package/.next/static/chunks/pages/_app-7d307437aca18ad4.js +1 -0
  86. package/.next/static/chunks/pages/_error-cb2a52f75f2162e2.js +1 -0
  87. package/.next/static/chunks/polyfills-42372ed130431b0a.js +1 -0
  88. package/.next/static/chunks/webpack-4a462cecab786e93.js +1 -0
  89. package/.next/static/css/bfd73afa11897439.css +3 -0
  90. package/.next/static/v_GdCj8lVweDVhmIhhEcM/_buildManifest.js +1 -0
  91. package/.next/static/v_GdCj8lVweDVhmIhhEcM/_ssgManifest.js +1 -0
  92. package/.next/trace +2 -0
  93. package/.next/types/app/api/get-avatars/route.ts +347 -0
  94. package/.next/types/app/api/start-session/route.ts +347 -0
  95. package/.next/types/app/layout.ts +84 -0
  96. package/.next/types/app/page.ts +84 -0
  97. package/.next/types/cache-life.d.ts +141 -0
  98. package/.next/types/package.json +1 -0
  99. package/.next/types/routes.d.ts +74 -0
  100. package/.next/types/validator.ts +88 -0
  101. package/README.md +241 -0
  102. package/app/api/config.ts +18 -0
  103. package/app/api/get-avatars/route.ts +117 -0
  104. package/app/api/start-session/route.ts +95 -0
  105. package/app/globals.css +3 -0
  106. package/app/layout.tsx +37 -0
  107. package/app/page.tsx +9 -0
  108. package/bin/cli.js +100 -0
  109. package/package.json +66 -0
  110. package/src/components/LiveAvatarSession.tsx +825 -0
  111. package/src/components/OpenClawDemo.tsx +399 -0
  112. package/src/gateway/client.ts +522 -0
  113. package/src/gateway/types.ts +83 -0
  114. package/src/liveavatar/context.tsx +750 -0
  115. package/src/liveavatar/index.ts +6 -0
  116. package/src/liveavatar/types.ts +10 -0
  117. package/src/liveavatar/useAvatarActions.ts +41 -0
  118. package/src/liveavatar/useChatHistory.ts +7 -0
  119. package/src/liveavatar/useSession.ts +37 -0
  120. package/src/liveavatar/useTextChat.ts +32 -0
  121. package/src/liveavatar/useVoiceChat.ts +70 -0
  122. package/tsconfig.json +40 -0
@@ -0,0 +1,825 @@
1
+ "use client";
2
+
3
+ import React, { useEffect, useRef, useState, useCallback } from "react";
4
+ import {
5
+ LiveAvatarContextProvider,
6
+ useSession,
7
+ useVoiceChat,
8
+ useLiveAvatarContext,
9
+ } from "../liveavatar";
10
+ import { SessionState } from "@heygen/liveavatar-web-sdk";
11
+ import { useAvatarActions } from "../liveavatar/useAvatarActions";
12
+ import { MessageSender } from "../liveavatar/types";
13
+ import { GatewayConnectionState } from "../gateway/types";
14
+
15
+ interface Avatar {
16
+ id: string;
17
+ name: string;
18
+ preview_url?: string;
19
+ default_voice?: {
20
+ id: string;
21
+ name: string;
22
+ };
23
+ is_custom?: boolean;
24
+ is_expired?: boolean;
25
+ }
26
+
27
+ // Audio level visualizer component
28
+ const AudioLevelMeter: React.FC<{
29
+ deviceId: string;
30
+ isActive: boolean;
31
+ }> = ({ deviceId, isActive }) => {
32
+ const [audioLevel, setAudioLevel] = useState(0);
33
+ const audioContextRef = useRef<AudioContext | null>(null);
34
+ const streamRef = useRef<MediaStream | null>(null);
35
+ const animationRef = useRef<number | null>(null);
36
+ const mountedRef = useRef(true);
37
+
38
+ useEffect(() => {
39
+ mountedRef.current = true;
40
+
41
+ if (!isActive || !deviceId || deviceId === "default") {
42
+ setAudioLevel(0);
43
+ return;
44
+ }
45
+
46
+ const cleanup = () => {
47
+ mountedRef.current = false;
48
+ if (animationRef.current) {
49
+ cancelAnimationFrame(animationRef.current);
50
+ animationRef.current = null;
51
+ }
52
+ if (streamRef.current) {
53
+ streamRef.current.getTracks().forEach((track) => track.stop());
54
+ streamRef.current = null;
55
+ }
56
+ if (audioContextRef.current) {
57
+ audioContextRef.current.close().catch(() => {});
58
+ audioContextRef.current = null;
59
+ }
60
+ };
61
+
62
+ const startAnalyser = async () => {
63
+ await new Promise((resolve) => setTimeout(resolve, 500));
64
+
65
+ if (!mountedRef.current) return;
66
+
67
+ try {
68
+ const stream = await navigator.mediaDevices.getUserMedia({
69
+ audio: { deviceId: { ideal: deviceId } },
70
+ });
71
+
72
+ if (!mountedRef.current) {
73
+ stream.getTracks().forEach((track) => track.stop());
74
+ return;
75
+ }
76
+
77
+ streamRef.current = stream;
78
+
79
+ const audioContext = new AudioContext();
80
+ audioContextRef.current = audioContext;
81
+
82
+ const source = audioContext.createMediaStreamSource(stream);
83
+ const analyser = audioContext.createAnalyser();
84
+ analyser.fftSize = 256;
85
+ analyser.smoothingTimeConstant = 0.3;
86
+ source.connect(analyser);
87
+
88
+ const dataArray = new Uint8Array(analyser.frequencyBinCount);
89
+
90
+ const updateLevel = () => {
91
+ if (!mountedRef.current) return;
92
+
93
+ analyser.getByteFrequencyData(dataArray);
94
+
95
+ let sum = 0;
96
+ for (let i = 0; i < dataArray.length; i++) {
97
+ sum += dataArray[i];
98
+ }
99
+ const average = sum / dataArray.length;
100
+ const normalizedLevel = Math.min(100, (average / 128) * 100);
101
+
102
+ setAudioLevel(normalizedLevel);
103
+ animationRef.current = requestAnimationFrame(updateLevel);
104
+ };
105
+
106
+ updateLevel();
107
+ } catch (err) {
108
+ console.error("Failed to start audio analyser:", err);
109
+ setAudioLevel(0);
110
+ }
111
+ };
112
+
113
+ startAnalyser();
114
+
115
+ return cleanup;
116
+ }, [deviceId, isActive]);
117
+
118
+ const bars = 5;
119
+ const barHeights = Array.from({ length: bars }, (_, i) => {
120
+ const threshold = (i + 1) * (100 / bars);
121
+ return audioLevel >= threshold ? 100 : (audioLevel / threshold) * 100;
122
+ });
123
+
124
+ return (
125
+ <div className="flex items-end gap-0.5 h-6">
126
+ {barHeights.map((height, i) => (
127
+ <div
128
+ key={i}
129
+ className="w-1 bg-green-500 rounded-full transition-all duration-75"
130
+ style={{
131
+ height: `${Math.max(4, height * 0.24)}px`,
132
+ opacity: audioLevel > 5 ? 1 : 0.3,
133
+ }}
134
+ />
135
+ ))}
136
+ </div>
137
+ );
138
+ };
139
+
140
+ // Microphone selector component
141
+ const MicrophoneSelector: React.FC<{
142
+ selectedDeviceId: string;
143
+ onDeviceChange: (deviceId: string) => void;
144
+ disabled?: boolean;
145
+ showAudioLevel?: boolean;
146
+ }> = ({ selectedDeviceId, onDeviceChange, disabled, showAudioLevel }) => {
147
+ const [devices, setDevices] = useState<MediaDeviceInfo[]>([]);
148
+ const [isOpen, setIsOpen] = useState(false);
149
+
150
+ useEffect(() => {
151
+ const getDevices = async () => {
152
+ try {
153
+ await navigator.mediaDevices.getUserMedia({ audio: true });
154
+ const allDevices = await navigator.mediaDevices.enumerateDevices();
155
+ const audioInputs = allDevices.filter(
156
+ (device) => device.kind === "audioinput"
157
+ );
158
+ setDevices(audioInputs);
159
+
160
+ if (audioInputs.length > 0 && selectedDeviceId === "default") {
161
+ onDeviceChange(audioInputs[0].deviceId);
162
+ }
163
+ } catch (err) {
164
+ console.error("Failed to get audio devices:", err);
165
+ }
166
+ };
167
+ getDevices();
168
+
169
+ navigator.mediaDevices.addEventListener("devicechange", getDevices);
170
+ return () => {
171
+ navigator.mediaDevices.removeEventListener("devicechange", getDevices);
172
+ };
173
+ }, [selectedDeviceId, onDeviceChange]);
174
+
175
+ const selectedDevice = devices.find((d) => d.deviceId === selectedDeviceId);
176
+ const displayName = selectedDevice?.label || "Select microphone";
177
+
178
+ return (
179
+ <div className="relative">
180
+ <button
181
+ onClick={() => !disabled && setIsOpen(!isOpen)}
182
+ disabled={disabled}
183
+ className={`flex items-center gap-2 bg-white/10 hover:bg-white/20 text-white px-3 py-2 rounded-lg text-sm transition-colors ${disabled ? "opacity-50 cursor-not-allowed" : ""}`}
184
+ >
185
+ <svg
186
+ className="w-4 h-4"
187
+ fill="none"
188
+ stroke="currentColor"
189
+ viewBox="0 0 24 24"
190
+ >
191
+ <path
192
+ strokeLinecap="round"
193
+ strokeLinejoin="round"
194
+ strokeWidth={2}
195
+ d="M19 11a7 7 0 01-7 7m0 0a7 7 0 01-7-7m7 7v4m0 0H8m4 0h4m-4-8a3 3 0 01-3-3V5a3 3 0 116 0v6a3 3 0 01-3 3z"
196
+ />
197
+ </svg>
198
+ <span className="max-w-[180px] truncate">{displayName}</span>
199
+ {showAudioLevel && selectedDeviceId !== "default" && (
200
+ <AudioLevelMeter
201
+ key={selectedDeviceId}
202
+ deviceId={selectedDeviceId}
203
+ isActive={true}
204
+ />
205
+ )}
206
+ <svg
207
+ className={`w-4 h-4 transition-transform ${isOpen ? "rotate-180" : ""}`}
208
+ fill="none"
209
+ stroke="currentColor"
210
+ viewBox="0 0 24 24"
211
+ >
212
+ <path
213
+ strokeLinecap="round"
214
+ strokeLinejoin="round"
215
+ strokeWidth={2}
216
+ d="M19 9l-7 7-7-7"
217
+ />
218
+ </svg>
219
+ </button>
220
+
221
+ {isOpen && (
222
+ <div className="absolute bottom-full left-0 mb-2 w-80 bg-white rounded-lg shadow-xl overflow-hidden z-50">
223
+ <div className="px-4 py-2 bg-gray-100 text-gray-600 text-sm font-medium border-b">
224
+ Select microphone
225
+ </div>
226
+ <div className="max-h-64 overflow-y-auto">
227
+ {devices.map((device) => (
228
+ <button
229
+ key={device.deviceId}
230
+ onClick={() => {
231
+ onDeviceChange(device.deviceId);
232
+ setIsOpen(false);
233
+ }}
234
+ className={`w-full px-4 py-3 text-left text-sm hover:bg-gray-50 transition-colors ${
235
+ device.deviceId === selectedDeviceId
236
+ ? "bg-blue-50 text-blue-700"
237
+ : "text-gray-800"
238
+ }`}
239
+ >
240
+ {device.label || `Microphone ${device.deviceId.slice(0, 8)}`}
241
+ </button>
242
+ ))}
243
+ </div>
244
+ </div>
245
+ )}
246
+ </div>
247
+ );
248
+ };
249
+
250
+ // Avatar selector component
251
+ const AvatarSelector: React.FC<{
252
+ onAvatarChange: (avatarId: string) => void;
253
+ isOpen: boolean;
254
+ onClose: () => void;
255
+ }> = ({ onAvatarChange, isOpen, onClose }) => {
256
+ const [avatars, setAvatars] = useState<Avatar[]>([]);
257
+ const [loading, setLoading] = useState(true);
258
+
259
+ useEffect(() => {
260
+ if (isOpen) {
261
+ const fetchAvatars = async () => {
262
+ setLoading(true);
263
+ try {
264
+ const res = await fetch("/api/get-avatars");
265
+ if (res.ok) {
266
+ const data = await res.json();
267
+ setAvatars(data.avatars || []);
268
+ }
269
+ } catch (err) {
270
+ console.error("Failed to fetch avatars:", err);
271
+ } finally {
272
+ setLoading(false);
273
+ }
274
+ };
275
+ fetchAvatars();
276
+ }
277
+ }, [isOpen]);
278
+
279
+ if (!isOpen) return null;
280
+
281
+ return (
282
+ <div className="fixed inset-0 bg-black/70 flex items-center justify-center z-50">
283
+ <div className="bg-gray-900 rounded-2xl p-6 max-w-2xl w-full mx-4 max-h-[80vh] overflow-hidden flex flex-col">
284
+ <div className="flex items-center justify-between mb-4">
285
+ <h2 className="text-xl font-semibold text-white">Change Avatar</h2>
286
+ <button
287
+ onClick={onClose}
288
+ className="text-gray-400 hover:text-white p-1"
289
+ >
290
+ <svg className="w-6 h-6" fill="none" stroke="currentColor" viewBox="0 0 24 24">
291
+ <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M6 18L18 6M6 6l12 12" />
292
+ </svg>
293
+ </button>
294
+ </div>
295
+
296
+ <p className="text-gray-400 text-sm mb-4">
297
+ Select an avatar to switch to. This will end your current session and start a new one.
298
+ </p>
299
+
300
+ {loading ? (
301
+ <div className="flex items-center justify-center py-12">
302
+ <div className="w-8 h-8 border-2 border-orange-500/30 border-t-orange-500 rounded-full animate-spin" />
303
+ </div>
304
+ ) : avatars.length === 0 ? (
305
+ <div className="text-center py-12 text-gray-400">
306
+ No avatars available
307
+ </div>
308
+ ) : (
309
+ <div className="overflow-y-auto flex-1">
310
+ <div className="grid grid-cols-4 gap-3">
311
+ {avatars.map((avatar) => (
312
+ <button
313
+ key={avatar.id}
314
+ onClick={() => {
315
+ onAvatarChange(avatar.id);
316
+ onClose();
317
+ }}
318
+ className="relative aspect-square rounded-lg overflow-hidden border-2 border-white/10 hover:border-orange-500/50 transition-all group"
319
+ >
320
+ {avatar.preview_url ? (
321
+ <img
322
+ src={avatar.preview_url}
323
+ alt={avatar.name}
324
+ className="w-full h-full object-cover"
325
+ />
326
+ ) : (
327
+ <div className="w-full h-full bg-gray-700 flex items-center justify-center">
328
+ <span className="text-gray-400 text-xs text-center px-1">
329
+ {avatar.name.slice(0, 10)}
330
+ </span>
331
+ </div>
332
+ )}
333
+ <div className="absolute inset-0 bg-black/0 group-hover:bg-black/40 transition-colors flex items-end">
334
+ <div className="w-full p-2 bg-gradient-to-t from-black/80 to-transparent opacity-0 group-hover:opacity-100 transition-opacity">
335
+ <span className="text-white text-xs truncate block">{avatar.name}</span>
336
+ </div>
337
+ </div>
338
+ {avatar.is_expired && (
339
+ <div className="absolute top-1 left-1 bg-red-500 text-white text-[10px] px-1 rounded">
340
+ Expired
341
+ </div>
342
+ )}
343
+ {avatar.is_custom && (
344
+ <div className="absolute top-1 right-1 bg-blue-500 text-white text-[10px] px-1 rounded">
345
+ Custom
346
+ </div>
347
+ )}
348
+ </button>
349
+ ))}
350
+ </div>
351
+ </div>
352
+ )}
353
+ </div>
354
+ </div>
355
+ );
356
+ };
357
+
358
+ // Gateway connection status indicator
359
+ const GatewayStatus: React.FC<{
360
+ state: GatewayConnectionState;
361
+ isProcessing: boolean;
362
+ isDemoMode: boolean;
363
+ }> = ({ state, isProcessing, isDemoMode }) => {
364
+ const getStatusColor = () => {
365
+ if (isProcessing) return "bg-yellow-500";
366
+ if (isDemoMode) return "bg-blue-500";
367
+ switch (state) {
368
+ case "connected":
369
+ return "bg-green-500";
370
+ case "connecting":
371
+ return "bg-yellow-500";
372
+ case "error":
373
+ return "bg-red-500";
374
+ default:
375
+ return "bg-gray-500";
376
+ }
377
+ };
378
+
379
+ const getStatusText = () => {
380
+ if (isProcessing) return "Processing...";
381
+ if (isDemoMode) return "Demo Mode";
382
+ switch (state) {
383
+ case "connected":
384
+ return "OpenClaw Connected";
385
+ case "connecting":
386
+ return "Connecting...";
387
+ case "error":
388
+ return "Connection Error";
389
+ default:
390
+ return "Disconnected";
391
+ }
392
+ };
393
+
394
+ return (
395
+ <div className="flex items-center gap-2 px-3 py-1.5 bg-black/30 rounded-full">
396
+ <span
397
+ className={`w-2 h-2 rounded-full ${getStatusColor()} ${isProcessing ? "animate-pulse" : ""}`}
398
+ />
399
+ <span className="text-xs text-white/80">{getStatusText()}</span>
400
+ </div>
401
+ );
402
+ };
403
+
404
+ // Chat transcript panel component with text input
405
+ const ChatPanel: React.FC = () => {
406
+ const { messages, gatewayState, isProcessingAgent, addTypedMessage, isDemoMode } = useLiveAvatarContext();
407
+ const messagesEndRef = useRef<HTMLDivElement>(null);
408
+ const [inputText, setInputText] = useState("");
409
+
410
+ // In demo mode or when connected, allow input
411
+ const isReady = isDemoMode || gatewayState === "connected";
412
+
413
+ useEffect(() => {
414
+ messagesEndRef.current?.scrollIntoView({ behavior: "smooth" });
415
+ }, [messages]);
416
+
417
+ const handleSendMessage = () => {
418
+ const text = inputText.trim();
419
+ if (!text) return;
420
+
421
+ addTypedMessage(text);
422
+ setInputText("");
423
+ };
424
+
425
+ const handleKeyDown = (e: React.KeyboardEvent<HTMLInputElement>) => {
426
+ if (e.key === "Enter" && !e.shiftKey) {
427
+ e.preventDefault();
428
+ handleSendMessage();
429
+ }
430
+ };
431
+
432
+ return (
433
+ <div className="flex flex-col h-full overflow-hidden bg-gray-900/50 rounded-2xl border border-white/10">
434
+ {/* Header with Gateway status */}
435
+ <div className="flex-shrink-0 px-4 py-3 border-b border-white/10 flex items-center justify-between">
436
+ <h3 className="text-white font-medium">Conversation</h3>
437
+ <GatewayStatus state={gatewayState} isProcessing={isProcessingAgent} isDemoMode={isDemoMode} />
438
+ </div>
439
+
440
+ {/* Messages */}
441
+ <div className="flex-1 min-h-0 overflow-y-auto p-4 space-y-3">
442
+ {messages.length === 0 ? (
443
+ <div className="text-gray-500 text-sm text-center py-8">
444
+ {isDemoMode ? (
445
+ <>Demo mode active. Type &quot;help&quot; to learn about this integration!</>
446
+ ) : gatewayState === "connected" ? (
447
+ <>Start speaking or type below to chat with your OpenClaw agent</>
448
+ ) : (
449
+ <>Connecting to OpenClaw Gateway...</>
450
+ )}
451
+ </div>
452
+ ) : (
453
+ messages.map((msg, index) => (
454
+ <div
455
+ key={index}
456
+ className={`flex ${msg.sender === MessageSender.USER ? "justify-end" : "justify-start"}`}
457
+ >
458
+ <div
459
+ className={`max-w-[90%] px-4 py-3 rounded-xl text-sm leading-relaxed ${
460
+ msg.sender === MessageSender.USER
461
+ ? "bg-blue-600 text-white"
462
+ : "bg-gray-700 text-gray-100"
463
+ }`}
464
+ >
465
+ <div className="text-xs opacity-70 mb-1.5 font-medium">
466
+ {msg.sender === MessageSender.USER ? "You" : "OpenClaw Agent"}
467
+ </div>
468
+ <div className="whitespace-pre-wrap">{msg.message}</div>
469
+ </div>
470
+ </div>
471
+ ))
472
+ )}
473
+ {isProcessingAgent && (
474
+ <div className="flex justify-start">
475
+ <div className="bg-gray-700 text-gray-100 px-3 py-2 rounded-lg text-sm">
476
+ <div className="flex items-center gap-2">
477
+ <div className="w-2 h-2 bg-blue-400 rounded-full animate-bounce" />
478
+ <div
479
+ className="w-2 h-2 bg-blue-400 rounded-full animate-bounce"
480
+ style={{ animationDelay: "0.1s" }}
481
+ />
482
+ <div
483
+ className="w-2 h-2 bg-blue-400 rounded-full animate-bounce"
484
+ style={{ animationDelay: "0.2s" }}
485
+ />
486
+ </div>
487
+ </div>
488
+ </div>
489
+ )}
490
+ <div ref={messagesEndRef} />
491
+ </div>
492
+
493
+ {/* Text input */}
494
+ <div className="flex-shrink-0 p-3 border-t border-white/10">
495
+ <div className="flex gap-2">
496
+ <input
497
+ type="text"
498
+ value={inputText}
499
+ onChange={(e) => setInputText(e.target.value)}
500
+ onKeyDown={handleKeyDown}
501
+ placeholder={isReady ? "Type a message..." : "Waiting for connection..."}
502
+ disabled={!isReady || isProcessingAgent}
503
+ className="flex-1 bg-gray-800 text-white text-sm px-3 py-2 rounded-lg border border-white/10 focus:border-orange-500/50 focus:outline-none disabled:opacity-50 disabled:cursor-not-allowed placeholder-gray-500"
504
+ />
505
+ <button
506
+ onClick={handleSendMessage}
507
+ disabled={!inputText.trim() || !isReady || isProcessingAgent}
508
+ className="bg-orange-500 hover:bg-orange-600 disabled:bg-gray-600 disabled:cursor-not-allowed text-white px-4 py-2 rounded-lg transition-colors"
509
+ >
510
+ <svg className="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
511
+ <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 19l9 2-9-18-9 18 9-2zm0 0v-8" />
512
+ </svg>
513
+ </button>
514
+ </div>
515
+ </div>
516
+ </div>
517
+ );
518
+ };
519
+
520
+ const LiveAvatarSessionComponent: React.FC<{
521
+ onSessionStopped: () => void;
522
+ onAvatarChange?: (avatarId: string) => void;
523
+ }> = ({ onSessionStopped }) => {
524
+ const [selectedMicId, setSelectedMicId] = useState("default");
525
+
526
+ const {
527
+ sessionState,
528
+ isStreamReady,
529
+ startSession,
530
+ stopSession,
531
+ connectionQuality,
532
+ attachElement,
533
+ sessionRef,
534
+ } = useSession();
535
+
536
+ const {
537
+ isAvatarTalking,
538
+ isUserTalking,
539
+ isMuted,
540
+ isActive,
541
+ mute,
542
+ unmute,
543
+ restartWithDevice,
544
+ } = useVoiceChat();
545
+
546
+ const { interrupt } = useAvatarActions("FULL");
547
+ const { gatewayState, isProcessingAgent } = useLiveAvatarContext();
548
+
549
+ const videoRef = useRef<HTMLVideoElement>(null);
550
+
551
+ useEffect(() => {
552
+ if (sessionState === SessionState.DISCONNECTED) {
553
+ onSessionStopped();
554
+ }
555
+ }, [sessionState, onSessionStopped]);
556
+
557
+ useEffect(() => {
558
+ if (isStreamReady && videoRef.current) {
559
+ attachElement(videoRef.current);
560
+ }
561
+ }, [attachElement, isStreamReady]);
562
+
563
+ useEffect(() => {
564
+ if (sessionState === SessionState.INACTIVE) {
565
+ startSession();
566
+ }
567
+ }, [startSession, sessionState]);
568
+
569
+ // Handle microphone device change
570
+ const handleMicChange = useCallback(
571
+ async (deviceId: string) => {
572
+ console.log("Changing microphone to:", deviceId);
573
+ setSelectedMicId(deviceId);
574
+
575
+ try {
576
+ const voiceChat = sessionRef.current?.voiceChat;
577
+ if (voiceChat) {
578
+ const result = await voiceChat.setDevice(deviceId);
579
+ if (!result && isActive) {
580
+ await restartWithDevice(deviceId);
581
+ }
582
+ }
583
+ } catch (err) {
584
+ console.error("Failed to set microphone device:", err);
585
+ if (isActive) {
586
+ await restartWithDevice(deviceId);
587
+ }
588
+ }
589
+ },
590
+ [sessionRef, isActive, restartWithDevice]
591
+ );
592
+
593
+ // Toggle mute
594
+ const handleMuteToggle = useCallback(async () => {
595
+ if (isMuted) {
596
+ await unmute();
597
+ } else {
598
+ await mute();
599
+ }
600
+ }, [isMuted, mute, unmute]);
601
+
602
+ // Calculate chat panel height
603
+ const videoContainerRef = useRef<HTMLDivElement>(null);
604
+ const [leftColumnHeight, setLeftColumnHeight] = useState<number>(0);
605
+
606
+ useEffect(() => {
607
+ const measureHeight = () => {
608
+ if (videoContainerRef.current) {
609
+ const leftColumn = videoContainerRef.current.parentElement;
610
+ if (leftColumn) {
611
+ setLeftColumnHeight(leftColumn.offsetHeight);
612
+ }
613
+ }
614
+ };
615
+ measureHeight();
616
+ window.addEventListener("resize", measureHeight);
617
+ return () => window.removeEventListener("resize", measureHeight);
618
+ }, [isStreamReady]);
619
+
620
+ return (
621
+ <div className="w-full max-w-6xl flex gap-4 py-4 px-4">
622
+ {/* Left side - Video and controls */}
623
+ <div className="flex-1 flex flex-col gap-4">
624
+ {/* Video container */}
625
+ <div
626
+ ref={videoContainerRef}
627
+ className="relative w-full aspect-video overflow-hidden rounded-2xl bg-gray-800 flex flex-col items-center justify-center"
628
+ >
629
+ <video
630
+ ref={videoRef}
631
+ autoPlay
632
+ playsInline
633
+ className="w-full h-full object-contain"
634
+ />
635
+
636
+ {/* Status overlay */}
637
+ {sessionState !== SessionState.CONNECTED && (
638
+ <div className="absolute inset-0 flex items-center justify-center bg-black/50">
639
+ <div className="text-white text-lg">
640
+ {sessionState === SessionState.CONNECTING && "Connecting..."}
641
+ {sessionState === SessionState.INACTIVE && "Starting..."}
642
+ {sessionState === SessionState.DISCONNECTING &&
643
+ "Disconnecting..."}
644
+ </div>
645
+ </div>
646
+ )}
647
+
648
+ {/* Bottom controls overlay */}
649
+ <div className="absolute bottom-4 right-4 flex items-center gap-2">
650
+ {/* End call button */}
651
+ <button
652
+ className="bg-red-500 hover:bg-red-600 text-white p-3 rounded-full transition-colors"
653
+ onClick={() => stopSession()}
654
+ title="End conversation"
655
+ >
656
+ <svg
657
+ className="w-5 h-5"
658
+ fill="none"
659
+ stroke="currentColor"
660
+ viewBox="0 0 24 24"
661
+ >
662
+ <path
663
+ strokeLinecap="round"
664
+ strokeLinejoin="round"
665
+ strokeWidth={2}
666
+ d="M6 18L18 6M6 6l12 12"
667
+ />
668
+ </svg>
669
+ </button>
670
+ </div>
671
+
672
+ {/* Talking indicators */}
673
+ <div className="absolute top-4 left-4 flex flex-col gap-2">
674
+ {isUserTalking && (
675
+ <div className="bg-green-500/80 text-white text-xs px-2 py-1 rounded-full flex items-center gap-1">
676
+ <span className="w-2 h-2 bg-white rounded-full animate-pulse" />
677
+ You&apos;re speaking
678
+ </div>
679
+ )}
680
+ {isAvatarTalking && (
681
+ <div className="bg-blue-500/80 text-white text-xs px-2 py-1 rounded-full flex items-center gap-1">
682
+ <span className="w-2 h-2 bg-white rounded-full animate-pulse" />
683
+ Agent speaking
684
+ </div>
685
+ )}
686
+ {isProcessingAgent && (
687
+ <div className="bg-yellow-500/80 text-white text-xs px-2 py-1 rounded-full flex items-center gap-1">
688
+ <span className="w-2 h-2 bg-white rounded-full animate-pulse" />
689
+ Thinking...
690
+ </div>
691
+ )}
692
+ </div>
693
+
694
+ {/* Connection quality */}
695
+ <div className="absolute top-4 right-4 px-3 py-1.5 rounded-full text-xs bg-black/50 text-white">
696
+ {connectionQuality}
697
+ </div>
698
+ </div>
699
+
700
+ {/* Control bar */}
701
+ <div className="w-full flex items-center justify-center gap-3">
702
+ {/* Mic selector with audio level */}
703
+ <MicrophoneSelector
704
+ selectedDeviceId={selectedMicId}
705
+ onDeviceChange={handleMicChange}
706
+ showAudioLevel={!isMuted}
707
+ />
708
+
709
+ {/* Single Mute/Unmute button */}
710
+ <button
711
+ onClick={handleMuteToggle}
712
+ className={`p-4 rounded-full transition-colors ${
713
+ isMuted
714
+ ? "bg-red-500 hover:bg-red-600"
715
+ : "bg-green-500 hover:bg-green-600"
716
+ }`}
717
+ title={isMuted ? "Click to unmute" : "Click to mute"}
718
+ >
719
+ {isMuted ? (
720
+ <svg
721
+ className="w-6 h-6 text-white"
722
+ fill="none"
723
+ stroke="currentColor"
724
+ viewBox="0 0 24 24"
725
+ >
726
+ <path
727
+ strokeLinecap="round"
728
+ strokeLinejoin="round"
729
+ strokeWidth={2}
730
+ d="M19 11a7 7 0 01-7 7m0 0a7 7 0 01-7-7m7 7v4m0 0H8m4 0h4m-4-8a3 3 0 01-3-3V5a3 3 0 116 0v6a3 3 0 01-3 3z"
731
+ />
732
+ <path
733
+ strokeLinecap="round"
734
+ strokeLinejoin="round"
735
+ strokeWidth={2}
736
+ d="M3 3l18 18"
737
+ />
738
+ </svg>
739
+ ) : (
740
+ <svg
741
+ className="w-6 h-6 text-white"
742
+ fill="none"
743
+ stroke="currentColor"
744
+ viewBox="0 0 24 24"
745
+ >
746
+ <path
747
+ strokeLinecap="round"
748
+ strokeLinejoin="round"
749
+ strokeWidth={2}
750
+ d="M19 11a7 7 0 01-7 7m0 0a7 7 0 01-7-7m7 7v4m0 0H8m4 0h4m-4-8a3 3 0 01-3-3V5a3 3 0 116 0v6a3 3 0 01-3 3z"
751
+ />
752
+ </svg>
753
+ )}
754
+ </button>
755
+
756
+ {/* Interrupt button */}
757
+ <button
758
+ onClick={() => interrupt()}
759
+ className="bg-orange-500 hover:bg-orange-600 text-white p-3 rounded-full transition-colors"
760
+ title="Interrupt avatar"
761
+ >
762
+ <svg
763
+ className="w-5 h-5"
764
+ fill="none"
765
+ stroke="currentColor"
766
+ viewBox="0 0 24 24"
767
+ >
768
+ <path
769
+ strokeLinecap="round"
770
+ strokeLinejoin="round"
771
+ strokeWidth={2}
772
+ d="M21 12a9 9 0 11-18 0 9 9 0 0118 0z"
773
+ />
774
+ <path
775
+ strokeLinecap="round"
776
+ strokeLinejoin="round"
777
+ strokeWidth={2}
778
+ d="M9 10a1 1 0 011-1h4a1 1 0 011 1v4a1 1 0 01-1 1h-4a1 1 0 01-1-1v-4z"
779
+ />
780
+ </svg>
781
+ </button>
782
+ </div>
783
+
784
+ {/* Status text */}
785
+ <div className="text-sm text-gray-400 text-center">
786
+ {gatewayState !== "connected" ? (
787
+ <span className="text-yellow-400">
788
+ Waiting for OpenClaw Gateway connection...
789
+ </span>
790
+ ) : isMuted ? (
791
+ "Microphone is muted - click to speak"
792
+ ) : (
793
+ "Speak to chat with your OpenClaw agent"
794
+ )}
795
+ </div>
796
+ </div>
797
+
798
+ {/* Right side - Chat panel (wider for better readability) */}
799
+ <div
800
+ className="w-[420px] flex flex-col overflow-hidden"
801
+ style={{
802
+ height: leftColumnHeight > 0 ? `${leftColumnHeight}px` : "500px",
803
+ }}
804
+ >
805
+ <ChatPanel />
806
+ </div>
807
+
808
+ </div>
809
+ );
810
+ };
811
+
812
+ export const LiveAvatarSession: React.FC<{
813
+ sessionAccessToken: string;
814
+ onSessionStopped: () => void;
815
+ onAvatarChange?: (avatarId: string) => void;
816
+ }> = ({ sessionAccessToken, onSessionStopped, onAvatarChange }) => {
817
+ return (
818
+ <LiveAvatarContextProvider sessionAccessToken={sessionAccessToken}>
819
+ <LiveAvatarSessionComponent
820
+ onSessionStopped={onSessionStopped}
821
+ onAvatarChange={onAvatarChange}
822
+ />
823
+ </LiveAvatarContextProvider>
824
+ );
825
+ };