@schoolio/player 1.4.0 → 1.4.1

package/dist/index.d.mts

@@ -237,6 +237,7 @@ declare class QuizApiClient {
         chatId: string | null;
         messages: ChatMessage[];
     }>;
+    getTextToSpeech(text: string, voice?: string): Promise<Blob>;
 }
 
 declare function checkAnswer(question: QuizQuestion, selectedAnswer: unknown): {

package/dist/index.d.ts

@@ -237,6 +237,7 @@ declare class QuizApiClient {
         chatId: string | null;
         messages: ChatMessage[];
     }>;
+    getTextToSpeech(text: string, voice?: string): Promise<Blob>;
 }
 
 declare function checkAnswer(question: QuizQuestion, selectedAnswer: unknown): {

package/dist/index.js

@@ -113,6 +113,23 @@ var QuizApiClient = class {
       `/api/external/question-chat/${questionId}/${childId}`
     );
   }
+  async getTextToSpeech(text, voice = "nova") {
+    const headers = {
+      "Content-Type": "application/json"
+    };
+    if (this.authToken) {
+      headers["Authorization"] = `Bearer ${this.authToken}`;
+    }
+    const response = await fetch(`${this.baseUrl}/api/external/tts`, {
+      method: "POST",
+      headers,
+      body: JSON.stringify({ text, voice })
+    });
+    if (!response.ok) {
+      throw new Error(`TTS request failed: HTTP ${response.status}`);
+    }
+    return response.blob();
+  }
 };
 
 // src/utils.ts
@@ -515,9 +532,13 @@ var panelStyles = {
   messageRow: {
     display: "flex"
   },
+  messageContent: {
+    display: "flex",
+    alignItems: "flex-end",
+    gap: "4px",
+    maxWidth: "85%"
+  },
   userMessage: {
-    maxWidth: "85%",
-    marginLeft: "auto",
     padding: "10px 14px",
     borderRadius: "16px 16px 4px 16px",
     backgroundColor: "#6721b0",
@@ -526,8 +547,6 @@ var panelStyles = {
     lineHeight: 1.4
   },
   assistantMessage: {
-    maxWidth: "85%",
-    marginRight: "auto",
     padding: "10px 14px",
     borderRadius: "16px 16px 16px 4px",
     backgroundColor: "#ffffff",
@@ -541,7 +560,8 @@ var panelStyles = {
     borderTop: "1px solid #e2e8f0",
     backgroundColor: "#ffffff",
     display: "flex",
-    gap: "8px"
+    gap: "8px",
+    alignItems: "center"
   },
   input: {
     flex: 1,
@@ -551,13 +571,11 @@ var panelStyles = {
     fontSize: "14px",
     outline: "none"
   },
-  sendButton: {
+  buttonBase: {
     width: "40px",
     height: "40px",
     borderRadius: "50%",
     border: "none",
-    backgroundColor: "#6721b0",
-    color: "#ffffff",
     cursor: "pointer",
     display: "flex",
     alignItems: "center",
@@ -565,10 +583,47 @@ var panelStyles = {
     fontSize: "16px",
     transition: "all 0.2s ease"
   },
+  sendButton: {
+    backgroundColor: "#6721b0",
+    color: "#ffffff"
+  },
   sendButtonDisabled: {
     backgroundColor: "#d1d5db",
     cursor: "not-allowed"
   },
+  micButton: {
+    backgroundColor: "#ffffff",
+    border: "1px solid #e2e8f0",
+    color: "#374151"
+  },
+  micButtonActive: {
+    backgroundColor: "#ef4444",
+    color: "#ffffff",
+    border: "none"
+  },
+  speakButton: {
+    width: "28px",
+    height: "28px",
+    borderRadius: "50%",
+    border: "none",
+    backgroundColor: "transparent",
+    color: "#9ca3af",
+    cursor: "pointer",
+    display: "flex",
+    alignItems: "center",
+    justifyContent: "center",
+    transition: "all 0.2s ease",
+    flexShrink: 0
+  },
+  speakButtonReady: {
+    color: "#22c55e"
+  },
+  speakButtonPlaying: {
+    color: "#6721b0"
+  },
+  speakButtonLoading: {
+    color: "#f97316"
+  },
   loadingDots: {
     display: "flex",
     alignItems: "center",
@@ -606,6 +661,33 @@ var STARTER_PROMPTS = [
   { id: "word_help", label: "I don't understand a word", message: "I don't understand a word in this question. Can you help?" },
   { id: "explain_again", label: "Explain this again", message: "Can you explain this question to me again in a different way?" }
 ];
+var MicIcon = () => /* @__PURE__ */ (0, import_jsx_runtime2.jsxs)("svg", { width: "18", height: "18", viewBox: "0 0 24 24", fill: "none", stroke: "currentColor", strokeWidth: "2", strokeLinecap: "round", strokeLinejoin: "round", children: [
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("path", { d: "M12 2a3 3 0 0 0-3 3v7a3 3 0 0 0 6 0V5a3 3 0 0 0-3-3Z" }),
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("path", { d: "M19 10v2a7 7 0 0 1-14 0v-2" }),
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("line", { x1: "12", x2: "12", y1: "19", y2: "22" })
+] });
+var MicOffIcon = () => /* @__PURE__ */ (0, import_jsx_runtime2.jsxs)("svg", { width: "18", height: "18", viewBox: "0 0 24 24", fill: "none", stroke: "currentColor", strokeWidth: "2", strokeLinecap: "round", strokeLinejoin: "round", children: [
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("line", { x1: "2", x2: "22", y1: "2", y2: "22" }),
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("path", { d: "M18.89 13.23A7.12 7.12 0 0 0 19 12v-2" }),
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("path", { d: "M5 10v2a7 7 0 0 0 12 5" }),
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("path", { d: "M15 9.34V5a3 3 0 0 0-5.68-1.33" }),
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("path", { d: "M9 9v3a3 3 0 0 0 5.12 2.12" }),
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("line", { x1: "12", x2: "12", y1: "19", y2: "22" })
+] });
+var VolumeIcon2 = () => /* @__PURE__ */ (0, import_jsx_runtime2.jsxs)("svg", { width: "16", height: "16", viewBox: "0 0 24 24", fill: "none", stroke: "currentColor", strokeWidth: "2", strokeLinecap: "round", strokeLinejoin: "round", children: [
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("polygon", { points: "11 5 6 9 2 9 2 15 6 15 11 19 11 5" }),
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("path", { d: "M15.54 8.46a5 5 0 0 1 0 7.07" }),
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("path", { d: "M19.07 4.93a10 10 0 0 1 0 14.14" })
+] });
+var SendIcon = () => /* @__PURE__ */ (0, import_jsx_runtime2.jsxs)("svg", { width: "18", height: "18", viewBox: "0 0 24 24", fill: "none", stroke: "currentColor", strokeWidth: "2", children: [
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("line", { x1: "22", y1: "2", x2: "11", y2: "13" }),
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("polygon", { points: "22 2 15 22 11 13 2 9 22 2" })
+] });
+var HelpIcon = () => /* @__PURE__ */ (0, import_jsx_runtime2.jsxs)("svg", { width: "48", height: "48", viewBox: "0 0 24 24", fill: "none", stroke: "#6721b0", strokeWidth: "2", children: [
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("circle", { cx: "12", cy: "12", r: "10" }),
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("path", { d: "M9.09 9a3 3 0 0 1 5.83 1c0 2-3 3-3 3" }),
+  /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("line", { x1: "12", y1: "17", x2: "12.01", y2: "17" })
+] });
 function QuestionChatPanel({
   apiClient,
   question,
@@ -620,13 +702,121 @@ function QuestionChatPanel({
   const [isLoading, setIsLoading] = (0, import_react2.useState)(false);
   const [chatId, setChatId] = (0, import_react2.useState)(null);
   const [hoveredButton, setHoveredButton] = (0, import_react2.useState)(null);
+  const [isListening, setIsListening] = (0, import_react2.useState)(false);
+  const [speakingIndex, setSpeakingIndex] = (0, import_react2.useState)(null);
+  const [audioReadyMap, setAudioReadyMap] = (0, import_react2.useState)(/* @__PURE__ */ new Map());
   const messagesContainerRef = (0, import_react2.useRef)(null);
   const messagesEndRef = (0, import_react2.useRef)(null);
+  const recognitionRef = (0, import_react2.useRef)(null);
+  const audioRef = (0, import_react2.useRef)(null);
+  const audioCacheRef = (0, import_react2.useRef)(/* @__PURE__ */ new Map());
+  const isSpeechSupported = typeof window !== "undefined" && (window.SpeechRecognition || window.webkitSpeechRecognition);
   const scrollToBottom = (0, import_react2.useCallback)(() => {
     if (messagesContainerRef.current) {
       messagesContainerRef.current.scrollTop = messagesContainerRef.current.scrollHeight;
     }
   }, []);
+  const preCacheAudio = (0, import_react2.useCallback)(async (text) => {
+    if (audioCacheRef.current.has(text)) {
+      setAudioReadyMap((prev) => new Map(prev).set(text, true));
+      return;
+    }
+    setAudioReadyMap((prev) => new Map(prev).set(text, false));
+    try {
+      const audioBlob = await apiClient.getTextToSpeech(text, "nova");
+      audioCacheRef.current.set(text, audioBlob);
+      setAudioReadyMap((prev) => new Map(prev).set(text, true));
+    } catch (error) {
+      console.error("Pre-cache TTS error:", error);
+    }
+  }, [apiClient]);
+  const startListening = (0, import_react2.useCallback)(() => {
+    const SpeechRecognitionAPI = window.SpeechRecognition || window.webkitSpeechRecognition;
+    if (!SpeechRecognitionAPI) {
+      console.log("Speech recognition not supported");
+      return;
+    }
+    const recognition = new SpeechRecognitionAPI();
+    recognition.continuous = true;
+    recognition.interimResults = true;
+    recognition.lang = "en-US";
+    recognition.onstart = () => {
+      console.log("Speech recognition started");
+      setIsListening(true);
+    };
+    recognition.onresult = (event) => {
+      let finalTranscript = "";
+      for (let i = 0; i < Object.keys(event.results).length; i++) {
+        const result = event.results[i];
+        if (result && result[0]) {
+          finalTranscript += result[0].transcript;
+        }
+      }
+      if (finalTranscript) {
+        setInputValue(finalTranscript);
+      }
+    };
+    recognition.onerror = (event) => {
+      console.log("Speech recognition error:", event.error);
+      if (event.error === "not-allowed") {
+        alert("Microphone access was denied. Please allow microphone access and try again.");
+      }
+      setIsListening(false);
+    };
+    recognition.onend = () => {
+      console.log("Speech recognition ended");
+      setIsListening(false);
+    };
+    recognitionRef.current = recognition;
+    try {
+      recognition.start();
+    } catch (e) {
+      console.log("Failed to start recognition:", e);
+      setIsListening(false);
+    }
+  }, []);
+  const stopListening = (0, import_react2.useCallback)(() => {
+    if (recognitionRef.current) {
+      recognitionRef.current.stop();
+      setIsListening(false);
+    }
+  }, []);
+  const speakMessage = (0, import_react2.useCallback)(async (text, index) => {
+    if (audioRef.current) {
+      audioRef.current.pause();
+      audioRef.current = null;
+    }
+    if (speakingIndex === index) {
+      setSpeakingIndex(null);
+      return;
+    }
+    setSpeakingIndex(index);
+    try {
+      let audioBlob;
+      const cachedBlob = audioCacheRef.current.get(text);
+      if (cachedBlob) {
+        audioBlob = cachedBlob;
+      } else {
+        audioBlob = await apiClient.getTextToSpeech(text, "nova");
+        audioCacheRef.current.set(text, audioBlob);
+      }
+      const audioUrl = URL.createObjectURL(audioBlob);
+      const audio = new Audio(audioUrl);
+      audioRef.current = audio;
+      audio.onended = () => {
+        setSpeakingIndex(null);
+        URL.revokeObjectURL(audioUrl);
+      };
+      audio.onerror = () => {
+        setSpeakingIndex(null);
+        URL.revokeObjectURL(audioUrl);
+      };
+      await audio.play();
+    } catch (error) {
+      console.error("TTS error:", error);
+      setSpeakingIndex(null);
+    }
+  }, [speakingIndex, apiClient]);
   (0, import_react2.useEffect)(() => {
     scrollToBottom();
   }, [messages, scrollToBottom]);
@@ -640,13 +830,14 @@ function QuestionChatPanel({
         if (history.chatId && history.messages.length > 0) {
           setChatId(history.chatId);
           setMessages(history.messages);
+          history.messages.filter((m) => m.role === "assistant").forEach((m) => preCacheAudio(m.content));
         }
       } catch (err) {
         console.error("Failed to load chat history:", err);
       }
     };
     loadHistory();
-  }, [question.id, childId, apiClient]);
+  }, [question.id, childId, apiClient, preCacheAudio]);
   const initializeChat = async () => {
     if (chatId) return chatId;
     try {
@@ -668,6 +859,9 @@ function QuestionChatPanel({
   };
   const sendMessage = async (messageText) => {
     if (!messageText.trim() || isLoading) return;
+    if (isListening) {
+      stopListening();
+    }
     setIsLoading(true);
     const userMsg = {
       role: "user",
@@ -687,15 +881,23 @@ function QuestionChatPanel({
         questionContext: question,
         childId
       });
-      setMessages((prev) => [...prev, response.assistantMessage]);
+      const assistantMessage = response.assistantMessage || {
+        role: "assistant",
+        content: "I'm here to help!",
+        timestamp: (/* @__PURE__ */ new Date()).toISOString()
+      };
+      setMessages((prev) => [...prev, assistantMessage]);
+      preCacheAudio(assistantMessage.content);
     } catch (err) {
       console.error("Failed to send message:", err);
+      const content = "Sorry, I'm having trouble right now. Please try again!";
       const errorMsg = {
         role: "assistant",
-        content: "Sorry, I'm having trouble right now. Please try again!",
+        content,
         timestamp: (/* @__PURE__ */ new Date()).toISOString()
       };
       setMessages((prev) => [...prev, errorMsg]);
+      preCacheAudio(content);
     } finally {
       setIsLoading(false);
     }
@@ -712,14 +914,14 @@ function QuestionChatPanel({
             0%, 60%, 100% { transform: translateY(0); }
             30% { transform: translateY(-4px); }
           }
+          @keyframes pulse {
+            0%, 100% { opacity: 1; }
+            50% { opacity: 0.5; }
+          }
         ` }),
     /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("div", { style: panelStyles.header, children: /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("span", { children: "Need Help?" }) }),
     /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("div", { ref: messagesContainerRef, style: panelStyles.messagesContainer, children: messages.length === 0 ? /* @__PURE__ */ (0, import_jsx_runtime2.jsxs)("div", { style: panelStyles.emptyState, children: [
-      /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("div", { style: panelStyles.helperIcon, children: /* @__PURE__ */ (0, import_jsx_runtime2.jsxs)("svg", { width: "48", height: "48", viewBox: "0 0 24 24", fill: "none", stroke: "#6721b0", strokeWidth: "2", children: [
-        /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("circle", { cx: "12", cy: "12", r: "10" }),
-        /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("path", { d: "M9.09 9a3 3 0 0 1 5.83 1c0 2-3 3-3 3" }),
-        /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("line", { x1: "12", y1: "17", x2: "12.01", y2: "17" })
-      ] }) }),
+      /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("div", { style: panelStyles.helperIcon, children: /* @__PURE__ */ (0, import_jsx_runtime2.jsx)(HelpIcon, {}) }),
       /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("div", { style: { fontSize: "14px", fontWeight: "500", marginBottom: "8px" }, children: "Hi! I'm your question helper" }),
       /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("div", { style: { fontSize: "13px", color: "#9ca3af" }, children: "Ask me if you need help understanding this question" }),
       /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("div", { style: { ...panelStyles.starterPrompts, marginTop: "16px" }, children: STARTER_PROMPTS.map((prompt) => /* @__PURE__ */ (0, import_jsx_runtime2.jsx)(
@@ -746,7 +948,26 @@ function QuestionChatPanel({
             ...panelStyles.messageRow,
             justifyContent: msg.role === "user" ? "flex-end" : "flex-start"
           },
-          children: /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("div", { style: msg.role === "user" ? panelStyles.userMessage : panelStyles.assistantMessage, children: msg.content })
+          children: /* @__PURE__ */ (0, import_jsx_runtime2.jsxs)("div", { style: {
+            ...panelStyles.messageContent,
+            flexDirection: msg.role === "user" ? "row-reverse" : "row"
+          }, children: [
+            /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("div", { style: msg.role === "user" ? panelStyles.userMessage : panelStyles.assistantMessage, children: msg.content }),
+            msg.role === "assistant" && /* @__PURE__ */ (0, import_jsx_runtime2.jsx)(
+              "button",
+              {
+                style: {
+                  ...panelStyles.speakButton,
+                  ...speakingIndex === idx ? panelStyles.speakButtonPlaying : audioReadyMap.get(msg.content) === true ? panelStyles.speakButtonReady : audioReadyMap.get(msg.content) === false ? panelStyles.speakButtonLoading : {},
+                  ...audioReadyMap.get(msg.content) === false ? { animation: "pulse 1.5s ease-in-out infinite" } : {}
+                },
+                onClick: () => speakMessage(msg.content, idx),
+                "data-testid": `button-speak-${idx}`,
+                title: "Listen to this message",
+                children: /* @__PURE__ */ (0, import_jsx_runtime2.jsx)(VolumeIcon2, {})
+              }
+            )
+          ] })
         },
         idx
       )),
@@ -765,26 +986,38 @@ function QuestionChatPanel({
           value: inputValue,
           onChange: (e) => setInputValue(e.target.value),
           onKeyPress: handleKeyPress,
-          placeholder: "Ask about this question...",
+          placeholder: isListening ? "Listening..." : "Ask about this question...",
           style: panelStyles.input,
-          disabled: isLoading,
+          disabled: isLoading || isListening,
           "data-testid": "input-chat-message"
         }
       ),
+      isSpeechSupported && /* @__PURE__ */ (0, import_jsx_runtime2.jsx)(
+        "button",
+        {
+          onClick: isListening ? stopListening : startListening,
+          disabled: isLoading,
+          style: {
+            ...panelStyles.buttonBase,
+            ...isListening ? panelStyles.micButtonActive : panelStyles.micButton
+          },
+          "data-testid": "button-voice-input",
+          title: isListening ? "Stop listening" : "Speak your question",
+          children: isListening ? /* @__PURE__ */ (0, import_jsx_runtime2.jsx)(MicOffIcon, {}) : /* @__PURE__ */ (0, import_jsx_runtime2.jsx)(MicIcon, {})
+        }
+      ),
       /* @__PURE__ */ (0, import_jsx_runtime2.jsx)(
         "button",
         {
           onClick: () => sendMessage(inputValue),
           disabled: isLoading || !inputValue.trim(),
           style: {
+            ...panelStyles.buttonBase,
             ...panelStyles.sendButton,
             ...isLoading || !inputValue.trim() ? panelStyles.sendButtonDisabled : {}
           },
           "data-testid": "button-send-chat",
-          children: /* @__PURE__ */ (0, import_jsx_runtime2.jsxs)("svg", { width: "18", height: "18", viewBox: "0 0 24 24", fill: "none", stroke: "currentColor", strokeWidth: "2", children: [
-            /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("line", { x1: "22", y1: "2", x2: "11", y2: "13" }),
-            /* @__PURE__ */ (0, import_jsx_runtime2.jsx)("polygon", { points: "22 2 15 22 11 13 2 9 22 2" })
-          ] })
+          children: /* @__PURE__ */ (0, import_jsx_runtime2.jsx)(SendIcon, {})
         }
       )
     ] })
@@ -1774,7 +2007,7 @@ function QuizPlayer({
       /* @__PURE__ */ (0, import_jsx_runtime3.jsxs)("div", { style: defaultStyles.quizContent, children: [
         /* @__PURE__ */ (0, import_jsx_runtime3.jsxs)("div", { style: { ...defaultStyles.question, position: "relative", paddingBottom: "40px" }, children: [
           /* @__PURE__ */ (0, import_jsx_runtime3.jsx)("div", { style: defaultStyles.questionText, children: /* @__PURE__ */ (0, import_jsx_runtime3.jsx)(TextToSpeech, { text: currentQuestion.question, inline: true, size: "md" }) }),
-          isExtraQuestion && !showFeedback && /* @__PURE__ */ (0, import_jsx_runtime3.jsxs)(
+          isExtraQuestion && /* @__PURE__ */ (0, import_jsx_runtime3.jsxs)(
             "button",
             {
               onClick: () => setShowSkipModal(true),
@@ -1815,7 +2048,7 @@ function QuizPlayer({
               ]
             }
           ),
-          !isExtraQuestion && !showFeedback && /* @__PURE__ */ (0, import_jsx_runtime3.jsxs)(
+          !isExtraQuestion && /* @__PURE__ */ (0, import_jsx_runtime3.jsxs)(
             "button",
             {
               onClick: () => setShowReportModal(true),