@nualang/nualang-ui-components 0.1.1419 → 0.1.1421

package/dist/hooks/useRecognition.js

@@ -9,7 +9,7 @@ const ENDPOINT = import.meta.env.REACT_APP_GCLOUD_SPEECH || "wss://speech-rec-se
 const socket = io(ENDPOINT, {
   withCredentials: true,
   autoConnect: false,
-  transports: ["websocket"] // use WebSocket only
+  transports: ["websocket", "polling"] // use WebSocket, fallback to polling if necessary
 });
 socket.on("connect_error", error => {
   // revert to classic upgrade
@@ -54,6 +54,24 @@ export default function useRecognition(props = {}) {
   const stopAudioRecordingTimeoutIdRef = useRef();
   const streamRef = useRef();
   const serverErrorRef = useRef();
+  const hasAlertedConnectionErrorRef = useRef(false);
+  // Guards against a second click landing during the async start-up window,
+  // before recognizing/isRecognizingDisabled state has re-rendered the button.
+  const isStartingRef = useRef(false);
+
+  // Lets us query, e.g., how often students end up on the polling fallback
+  // (a proxy for "couldn't connect via websocket") or hit connect errors.
+  function trackConnectionDiagnostics(eventName, extra = {}) {
+    if (typeof trackRecommendedEvent !== "function") {
+      return;
+    }
+    trackRecommendedEvent(eventName, {
+      A1_browser: `${getBrowserInfo(navigator.userAgent)}`,
+      A2_machine_info: `${getMachineInfo(navigator.userAgent)}`,
+      A3_exercise: gameId ? `${exerciseName}-${gameId}` : `${exerciseName}`,
+      ...extra
+    });
+  }
   async function loadModule() {
     try {
       await audioContextRef.current.audioWorklet.addModule(`/worklets/recording-processor-v2.js`);
@@ -72,7 +90,9 @@ export default function useRecognition(props = {}) {
         });
         await loadModule();
       }
-      audioContextRef.current.resume();
+      // Must be awaited - an unawaited rejection here would leave the
+      // context suspended with no error surfaced, silently killing capture.
+      await audioContextRef.current.resume();
     } catch (e) {
       console.error(`Sorry, there was an issue setting up the audio context`, e);
       throw e;
@@ -80,13 +100,71 @@ export default function useRecognition(props = {}) {
   }
   useEffect(() => {
     function onConnect() {
+      hasAlertedConnectionErrorRef.current = false;
       setIsSocketConnected(true);
+      // socket.io.engine.transport.name is "polling" here only when the
+      // websocket attempt failed and it fell back - the clearest signal we
+      // have that a student couldn't connect via websocket.
+      trackConnectionDiagnostics("speech_recognition_socket_connected", {
+        A4_transport: socket.io.engine.transport.name
+      });
     }
     function onDisconnect() {
       setIsSocketConnected(false);
+      // The server-side recognize stream/session is gone with the connection
+      // (no state recovery), so leaving recognizing:true would keep streaming
+      // audio into a void with no feedback until the 10s no-transcript timer.
+      setRecognitionState(prevState => {
+        if (stopAudioRecordingTimeoutIdRef.current) {
+          clearTimeout(stopAudioRecordingTimeoutIdRef.current);
+          stopAudioRecordingTimeoutIdRef.current = null;
+        }
+        // A start() in flight (isStartingRef/isRecognizingDisabled set, but
+        // recognizing not yet true) needs the same unwind as an active
+        // recording - otherwise it's neither idle nor recovered, and the
+        // getUserMedia continuation below would still flip recognizing:true
+        // once it resolves against a dead socket.
+        const isStarting = isStartingRef.current || prevState.isRecognizingDisabled;
+        if (!prevState.recognizing && !isStarting) {
+          // Truly idle - nothing to unwind, but the mic button is about to go
+          // disabled - without this the student just finds a dead button
+          // later with no explanation. Guarded so it only fires once per
+          // disconnected streak.
+          if (!hasAlertedConnectionErrorRef.current) {
+            hasAlertedConnectionErrorRef.current = true;
+            recognitionAlert("Connection to the speech recognition service was lost. The microphone will be unavailable until it reconnects.");
+          }
+          return prevState;
+        }
+        isStartingRef.current = false;
+        closeAll(true);
+        if (!hasAlertedConnectionErrorRef.current) {
+          hasAlertedConnectionErrorRef.current = true;
+          recognitionAlert(prevState.recognizing ? "Connection to the speech recognition service was lost while recording. Please try again." : "Connection to the speech recognition service was lost while starting the microphone. Please try again.");
+        }
+        return {
+          ...prevState,
+          recognizing: false,
+          isRecognizingDisabled: false
+        };
+      });
+    }
+    function onConnectError(error) {
+      console.error(error);
+      trackConnectionDiagnostics("speech_recognition_connect_error", {
+        A4_error_message: `${error?.message}`
+      });
+      // Without this, a blocked/failed connection (e.g. a school network
+      // firewall blocking the websocket) leaves the mic button silently
+      // disabled with no feedback to the student.
+      if (!hasAlertedConnectionErrorRef.current) {
+        hasAlertedConnectionErrorRef.current = true;
+        recognitionAlert("Could not connect to the speech recognition service. Please check your network connection (some school Wi-Fi/firewalls block this) and refresh the page.");
+      }
     }
     socket.on("connect", onConnect);
     socket.on("disconnect", onDisconnect);
+    socket.on("connect_error", onConnectError);
 
     // no-op if the socket is already connected
     socket.connect();
@@ -94,14 +172,19 @@ export default function useRecognition(props = {}) {
       socket.disconnect();
       socket.off("connect", onConnect);
       socket.off("disconnect", onDisconnect);
-      closeAll();
+      socket.off("connect_error", onConnectError);
+      closeAll(false, true);
     };
   }, []);
 
   /**
    * Stops recording and closes everything down. Runs on error or on stop.
+   * By default the AudioContext is only suspended, not closed, so the next
+   * start() can reuse it instead of re-fetching/re-registering the audio
+   * worklet module (a source of noticeable start-up lag). Pass fullClose
+   * when the hook itself is unmounting and the context should be released.
    */
-  function closeAll(keepSocketAlive) {
+  function closeAll(keepSocketAlive, fullClose = false) {
     // Clear the listeners (prevents issue if opening and closing repeatedly)
     if (keepSocketAlive) {
       socket.off("speechData");
@@ -114,6 +197,7 @@ export default function useRecognition(props = {}) {
         track.stop();
       });
     }
+    streamRef.current = null;
     if (processorRef.current) {
       if (audioInputRef.current) {
         try {
@@ -122,19 +206,30 @@ export default function useRecognition(props = {}) {
           logger.warn(`Attempt to disconnect input failed: ${error}`);
         }
       }
-      processorRef.current.disconnect(audioContextRef.current.destination);
+      try {
+        processorRef.current.disconnect(audioContextRef.current?.destination);
+      } catch (error) {
+        logger.warn(`Attempt to disconnect processor failed: ${error}`);
+      }
+    }
+    audioInputRef.current = null;
+    processorRef.current = null;
+    if (!audioContextRef.current) {
+      return;
     }
-    if (audioContextRef.current) {
+    if (fullClose) {
       audioContextRef.current.close().then(function () {
-        audioInputRef.current = null;
-        processorRef.current = null;
         audioContextRef.current = null;
       });
+    } else {
+      audioContextRef.current.suspend().catch(error => {
+        logger.warn(`Failed to suspend audio context: ${error}`);
+      });
     }
   }
-  function recognitionAlert(message) {
+  function recognitionAlert(message, severity = "error") {
     if (openSnackbar) {
-      openSnackbar(message, "error");
+      openSnackbar(message, severity);
     } else {
       alert(message);
     }
@@ -148,7 +243,15 @@ export default function useRecognition(props = {}) {
   const initRecording = async (gcloudParams, onData, onError) => {
     try {
       await setupAudioContext();
-      const handleSuccess = function (stream) {
+      const handleSuccess = async function (stream) {
+        streamRef.current = stream;
+        if (!socket.connected) {
+          // Dropped while getUserMedia was resolving - don't buffer a stale
+          // startGoogleCloudStream for a session the server already tore
+          // down. onDisconnect owns the alert/state reset.
+          closeAll(true);
+          return;
+        }
         const gcloudConfig = {
           config: {
             encoding: "LINEAR16",
@@ -167,12 +270,28 @@ export default function useRecognition(props = {}) {
         };
         socket.emit("startGoogleCloudStream", gcloudConfig, version); // init socket Google Speech Connection
 
-        streamRef.current = stream;
         audioInputRef.current = audioContextRef.current.createMediaStreamSource(stream);
         serverErrorRef.current = null;
         processorRef.current = new AudioWorkletNode(audioContextRef.current, "recording-processor-v2");
         processorRef.current.connect(audioContextRef.current.destination);
-        audioContextRef.current.resume();
+        try {
+          // Awaited so a rejection reaches the getUserMedia .catch() below
+          // instead of vanishing as an unhandled rejection.
+          await audioContextRef.current.resume();
+        } catch (resumeError) {
+          // stream/audioInput/processor are already live at this point -
+          // without this the mic track and worklet node would leak past the
+          // failure that the outer .catch() alerts and resets state for.
+          closeAll(true);
+          throw resumeError;
+        }
+        if (!socket.connected) {
+          // Dropped mid-resume; onDisconnect has already (or is about to)
+          // close these same resources, so stop instead of wiring nodes onto
+          // a torn-down session.
+          closeAll(true);
+          return;
+        }
         audioInputRef.current.connect(processorRef.current);
         processorRef.current.port.onmessage = event => {
           const audioData = event.data;
@@ -188,50 +307,86 @@ export default function useRecognition(props = {}) {
           socket.emit("binaryData", result, version);
         };
         if (window.MediaRecorder) {
-          mediaRecorderRef.current = new MediaRecorder(stream);
-          mediaRecorderRef.current.ondataavailable = e => {
-            const audioChunks = [];
-            audioChunks.push(e.data);
-            const audioBlob = new Blob(audioChunks, {
-              type: mediaRecorderRef.current.mimeType
-            });
-            const audioUrl = URL.createObjectURL(audioBlob);
-            setRecognitionState(prevState => ({
-              ...prevState,
-              attemptAudioURL: audioUrl,
-              audioBlob: audioBlob
-            }));
-          };
-          mediaRecorderRef.current.start();
+          // This only powers the "listen back to your attempt" playback
+          // feature, not recognition itself. A codec/construction failure
+          // here (seen in some embedded/older WebKit webviews) must not
+          // abort the recording flow that already started above.
+          try {
+            mediaRecorderRef.current = new MediaRecorder(stream);
+            mediaRecorderRef.current.ondataavailable = e => {
+              const audioChunks = [];
+              audioChunks.push(e.data);
+              const audioBlob = new Blob(audioChunks, {
+                type: mediaRecorderRef.current.mimeType
+              });
+              const audioUrl = URL.createObjectURL(audioBlob);
+              setRecognitionState(prevState => ({
+                ...prevState,
+                attemptAudioURL: audioUrl,
+                audioBlob: audioBlob
+              }));
+            };
+            mediaRecorderRef.current.start();
+          } catch (mediaRecorderError) {
+            console.error("Failed to start MediaRecorder for attempt playback", mediaRecorderError);
+            mediaRecorderRef.current = null;
+          }
         }
-        if (/Android|webOS|Safari|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent)) {
+        if (
+        // Excludes bare "Safari", which also matches every desktop
+        // Chrome/Edge/Firefox UA string and was adding this delay for
+        // almost all browsers, not just mobile ones.
+        /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent)) {
           // wait half second to indicate recording, this helps prevent speech being lost at the start
           return new Promise(resolve => setTimeout(resolve, 500));
         }
       };
       if (navigator && navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
         navigator.mediaDevices.getUserMedia(constraints).then(handleSuccess).then(() => {
+          isStartingRef.current = false;
+          // socket.connected is checked directly (not the isSocketConnected
+          // state, which may not have re-rendered yet) so a disconnect that
+          // landed mid-setup can't be raced into recognizing:true - onDisconnect
+          // already unwound this startup, so just release what handleSuccess
+          // just created.
+          if (!socket.connected) {
+            closeAll(true);
+            return;
+          }
           if (!serverErrorRef.current) {
             setRecognitionState(prevState => ({
               ...prevState,
-              recognizing: true
+              recognizing: true,
+              isRecognizingDisabled: false
             }));
           }
         }).catch(err => {
           /* handle the error */
           console.error(err);
-          if (["Permission denied", "The request is not allowed by the user agent or the platform in the current context, possibly because the user denied permission.", "The request is not allowed by the user agent or the platform in the current context."].includes(err?.message)) {
-            recognitionAlert("Permission denied, Please allow microphone access and refresh the page.");
+          // Match on err.name (the DOMException standard) rather than the
+          // message text, which varies by browser/locale and previously
+          // meant most getUserMedia failures (e.g. a Permissions-Policy
+          // block when embedded in an iframe/webview such as an LMS app)
+          // were logged but never surfaced to the student.
+          const isEmbedded = typeof window !== "undefined" && window.self !== window.top;
+          if (["NotAllowedError", "PermissionDeniedError", "SecurityError"].includes(err?.name)) {
+            recognitionAlert(isEmbedded ? "Microphone access was blocked. If you're inside another app, make sure it allows microphone access." : "Permission denied, Please allow microphone access and refresh the page.");
+          } else {
+            recognitionAlert("Could not access the microphone. Please refresh the page and try again.");
           }
+          isStartingRef.current = false;
           setRecognitionState(prevState => ({
             ...prevState,
-            recognizing: false
+            recognizing: false,
+            isRecognizingDisabled: false
           }));
         });
       } else {
+        isStartingRef.current = false;
         setRecognitionState(prevState => ({
           ...prevState,
-          recognizing: false
+          recognizing: false,
+          isRecognizingDisabled: false
         }));
         recognitionAlert("Speech recognition not supported, please try another browser.");
       }
@@ -250,6 +405,12 @@ export default function useRecognition(props = {}) {
         }
       });
     } catch (error) {
+      isStartingRef.current = false;
+      setRecognitionState(prevState => ({
+        ...prevState,
+        recognizing: false,
+        isRecognizingDisabled: false
+      }));
       if (onError) {
         onError(error);
       }
@@ -280,7 +441,7 @@ export default function useRecognition(props = {}) {
     };
     trackRecommendedEvent("speech_recognition_no_transcript", eventBody);
   };
-  const stopRecording = (keepSocketAlive, triggerEvent) => {
+  const stopRecording = (keepSocketAlive, triggerEvent, fullClose = false) => {
     // indicate end stream to trigger interim transcript to return as final transcript
     if (socket) {
       indicateEndStream();
@@ -289,7 +450,7 @@ export default function useRecognition(props = {}) {
       mediaRecorderRef.current.stop();
     }
     setTimeout(() => {
-      closeAll(keepSocketAlive);
+      closeAll(keepSocketAlive, fullClose);
       setRecognitionState(prevState => ({
         ...prevState,
         isRecognizingDisabled: false
@@ -305,7 +466,7 @@ export default function useRecognition(props = {}) {
   };
   useEffect(() => {
     return () => {
-      stopRecording(false);
+      stopRecording(false, false, true);
     };
   }, []);
   const startRecognition = (lang, speechContexts, model) => {
@@ -408,6 +569,16 @@ export default function useRecognition(props = {}) {
     stop();
   };
   const start = (lang, speechContexts, model) => {
+    // Ignore a repeat click while a start is already in flight or once
+    // recognizing - a ref check because it must be synchronous, unlike state.
+    if (isStartingRef.current || recognitionState.recognizing) {
+      return;
+    }
+    isStartingRef.current = true;
+    setRecognitionState(prevState => ({
+      ...prevState,
+      isRecognizingDisabled: true
+    }));
     stopAudioRecordingTimeoutIdRef.current = setTimeout(() => {
       stop();
     }, audioRecordingLimit);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nualang/nualang-ui-components",
-  "version": "0.1.1419",
+  "version": "0.1.1421",
   "type": "module",
   "main": "dist/index.js",
   "files": [
@@ -177,6 +177,7 @@
     "build-storybook": "storybook build",
     "clean": "rimraf dist",
     "compile": "pnpm run clean && cross-env NODE_ENV=production babel src/components --out-dir dist --copy-files --no-copy-ignored",
+    "watch": "pnpm run clean && cross-env NODE_ENV=production babel src/components --out-dir dist --copy-files --no-copy-ignored --watch",
     "version": "pnpm run compile && git add -A",
     "postversion": "pnpm publish && git push && git push --tags",
     "localpack": "pnpm run compile && pnpm pack",