openclaw-liveavatar 1.0.0

package/src/liveavatar/index.ts

@@ -0,0 +1,6 @@
+export { LiveAvatarContextProvider, useLiveAvatarContext } from "./context";
+export * from "./types";
+export { useChatHistory } from "./useChatHistory";
+export { useSession } from "./useSession";
+export { useVoiceChat } from "./useVoiceChat";
+export { useTextChat } from "./useTextChat";

package/src/liveavatar/types.ts

@@ -0,0 +1,10 @@
+export enum MessageSender {
+  USER = "user",
+  AVATAR = "avatar",
+}
+
+export interface LiveAvatarSessionMessage {
+  sender: MessageSender;
+  message: string;
+  timestamp: number;
+}

package/src/liveavatar/useAvatarActions.ts

@@ -0,0 +1,41 @@
+import { useCallback } from "react";
+import { useLiveAvatarContext } from "./context";
+
+export const useAvatarActions = (mode: "FULL" | "CUSTOM") => {
+  const { sessionRef } = useLiveAvatarContext();
+
+  const interrupt = useCallback(() => {
+    return sessionRef.current.interrupt();
+  }, [sessionRef]);
+
+  const repeat = useCallback(
+    async (message: string) => {
+      if (mode === "FULL") {
+        return sessionRef.current.repeat(message);
+      } else if (mode === "CUSTOM") {
+        const res = await fetch("/api/elevenlabs-text-to-speech", {
+          method: "POST",
+          body: JSON.stringify({ text: message }),
+        });
+        const { audio } = await res.json();
+        return sessionRef.current.repeatAudio(audio);
+      }
+    },
+    [sessionRef, mode],
+  );
+
+  const startListening = useCallback(() => {
+    return sessionRef.current.startListening();
+  }, [sessionRef]);
+
+  const stopListening = useCallback(() => {
+    return sessionRef.current.stopListening();
+  }, [sessionRef]);
+
+  return {
+    interrupt,
+    repeat,
+    startListening,
+    stopListening,
+  };
+};

package/src/liveavatar/useChatHistory.ts

@@ -0,0 +1,7 @@
+import { useLiveAvatarContext } from "./context";
+
+export const useChatHistory = () => {
+  const { messages } = useLiveAvatarContext();
+
+  return messages;
+};

package/src/liveavatar/useSession.ts

@@ -0,0 +1,37 @@
+import { useCallback } from "react";
+import { useLiveAvatarContext } from "./context";
+
+export const useSession = () => {
+  const { sessionRef, sessionState, isStreamReady, connectionQuality } =
+    useLiveAvatarContext();
+
+  const startSession = useCallback(async () => {
+    return await sessionRef.current.start();
+  }, [sessionRef]);
+
+  const stopSession = useCallback(async () => {
+    return await sessionRef.current.stop();
+  }, [sessionRef]);
+
+  const keepAlive = useCallback(async () => {
+    return await sessionRef.current.keepAlive();
+  }, [sessionRef]);
+
+  const attachElement = useCallback(
+    (element: HTMLMediaElement) => {
+      return sessionRef.current.attach(element);
+    },
+    [sessionRef],
+  );
+
+  return {
+    sessionState,
+    isStreamReady,
+    connectionQuality,
+    startSession,
+    stopSession,
+    keepAlive,
+    attachElement,
+    sessionRef,
+  };
+};

package/src/liveavatar/useTextChat.ts

@@ -0,0 +1,32 @@
+import { useCallback } from "react";
+import { useLiveAvatarContext } from "./context";
+
+export const useTextChat = (mode: "FULL" | "CUSTOM") => {
+  const { sessionRef } = useLiveAvatarContext();
+
+  const sendMessage = useCallback(
+    async (message: string) => {
+      if (mode === "FULL") {
+        return sessionRef.current.message(message);
+      } else if (mode === "CUSTOM") {
+        const response = await fetch("/api/openai-chat-complete", {
+          method: "POST",
+          body: JSON.stringify({ message }),
+        });
+        const { response: chatResponseText } = await response.json();
+        const res = await fetch("/api/elevenlabs-text-to-speech", {
+          method: "POST",
+          body: JSON.stringify({ text: chatResponseText }),
+        });
+        const { audio } = await res.json();
+        // Have the avatar repeat the audio
+        return sessionRef.current.repeatAudio(audio);
+      }
+    },
+    [sessionRef, mode],
+  );
+
+  return {
+    sendMessage,
+  };
+};

package/src/liveavatar/useVoiceChat.ts

@@ -0,0 +1,70 @@
+import { useCallback, useMemo } from "react";
+import { useLiveAvatarContext } from "./context";
+import { VoiceChatState } from "@heygen/liveavatar-web-sdk";
+
+export const useVoiceChat = () => {
+  const {
+    sessionRef,
+    isMuted,
+    voiceChatState,
+    isUserTalking,
+    isAvatarTalking,
+  } = useLiveAvatarContext();
+
+  const mute = useCallback(async () => {
+    return await sessionRef.current.voiceChat.mute();
+  }, [sessionRef]);
+
+  const unmute = useCallback(async () => {
+    return await sessionRef.current.voiceChat.unmute();
+  }, [sessionRef]);
+
+  const start = useCallback(async (deviceId?: string) => {
+    return await sessionRef.current.voiceChat.start(
+      deviceId ? { deviceId, defaultMuted: false } : { defaultMuted: false }
+    );
+  }, [sessionRef]);
+
+  const stop = useCallback(() => {
+    return sessionRef.current.voiceChat.stop();
+  }, [sessionRef]);
+
+  // Restart voice chat with a new device
+  const restartWithDevice = useCallback(async (deviceId: string) => {
+    console.log("Restarting voice chat with device:", deviceId);
+    try {
+      // Stop current voice chat
+      sessionRef.current.voiceChat.stop();
+      // Wait a bit for cleanup
+      await new Promise(resolve => setTimeout(resolve, 100));
+      // Start with new device
+      await sessionRef.current.voiceChat.start({ deviceId, defaultMuted: false });
+      console.log("Voice chat restarted successfully");
+      return true;
+    } catch (err) {
+      console.error("Failed to restart voice chat:", err);
+      return false;
+    }
+  }, [sessionRef]);
+
+  const isLoading = useMemo(() => {
+    return voiceChatState === VoiceChatState.STARTING;
+  }, [voiceChatState]);
+
+  const isActive = useMemo(() => {
+    return voiceChatState === VoiceChatState.ACTIVE;
+  }, [voiceChatState]);
+
+  return {
+    mute,
+    unmute,
+    start,
+    stop,
+    restartWithDevice,
+    isLoading,
+    isActive,
+    isMuted,
+    isUserTalking,
+    isAvatarTalking,
+  };
+};

package/tsconfig.json

@@ -0,0 +1,40 @@
+{
+  "compilerOptions": {
+    "lib": [
+      "dom",
+      "dom.iterable",
+      "esnext"
+    ],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": [
+        "./*"
+      ]
+    },
+    "target": "ES2017"
+  },
+  "include": [
+    "next-env.d.ts",
+    "**/*.ts",
+    "**/*.tsx",
+    ".next/types/**/*.ts"
+  ],
+  "exclude": [
+    "node_modules"
+  ]
+}