@magmamath/students-features 0.9.110 → 0.9.111-rc-0

package/src/features/chatbot/TTS_HIGHLIGHTING.md

@@ -0,0 +1,109 @@
+# TTS Word-by-Word Highlighting
+
+## Overview
+
+Word-by-word text highlighting synchronized with text-to-speech playback in the chatbot. The feature is optional and enabled via props.
+
+## How It Works
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│  1. User clicks TTS button on a message                         │
+│                           ↓                                     │
+│  2. Text is converted to SSML with word marks                   │
+│     "Hello world" → <speak><mark name="w_0"/>Hello              │
+│                            <mark name="w_1"/>world</speak>      │
+│                           ↓                                     │
+│  3. Google TTS API returns audio + timepoints                   │
+│     [{ markName: "w_0", timeSeconds: 0.0 },                     │
+│      { markName: "w_1", timeSeconds: 0.5 }]                     │
+│                           ↓                                     │
+│  4. Audio plays, ontimeupdate fires (~4x/sec)                   │
+│                           ↓                                     │
+│  5. Current time → find matching timepoint → word index         │
+│                           ↓                                     │
+│  6. $highlightedWordIndex updates → UI re-renders highlighted   │
+└─────────────────────────────────────────────────────────────────┘
+```
+
+## Usage
+
+```tsx
+// In students-web model setup
+chatModel.bot.setUpT2S({
+  audioProvider: createT2SAudioProvider(),
+  textSanitizer: removeKatexPartFromTextToSpeech,
+  isHighlightEnabled: true,  // Enable highlighting
+})
+
+// In component
+<Chatbot
+  model={chatModel.bot}
+  isHighlightEnabled={true}
+  highlightConfig={{
+    MessageRenderer: ChatbotHighlightedMessage,
+  }}
+/>
+```
+
+## Files Modified
+
+### students-features
+
+| File | Changes |
+|------|---------|
+| `types/t2s.types.ts` | Added `onTimeUpdate`, `getCurrentTime` to AudioProvider; added `HighlightConfig` type |
+| `types/api.types.ts` | Added `ssml` option to GoogleText2SpeechPayload input |
+| `helpers/textToSSML.ts` | NEW - Converts text to SSML with word marks |
+| `model/t2s.ts` | Added `$highlightedWordIndex`, `$timepoints`, `$time` stores; SSML/timepoint handling |
+| `components/Chatbot.tsx` | Added `isHighlightEnabled`, `highlightConfig` props |
+| `components/Chat/Chat.tsx` | Passes highlight state to ChatMessage |
+| `components/ChatMessage/*` | Passes highlight props through component chain |
+| `components/ChatMessage/MessageContent.tsx` | Renders custom MessageRenderer when provided |
+
+### students-web
+
+| File | Changes |
+|------|---------|
+| `chatbot/helpers.ts` | Added `onTimeUpdate`, `getCurrentTime` to AudioProvider |
+| `chatbot/components/ChatbotHighlightedMessage.tsx` | NEW - Renders words with highlight styling |
+| `chatbot/components/ChatbotHighlightedMessage.module.scss` | NEW - Yellow highlight styles |
+| `chatbot/model/model.ts` | Enabled `isHighlightEnabled: true` |
+| `PracticeMode/chatbot/PracticeChatbot.tsx` | Passes highlightConfig with MessageRenderer |
+
+## Key Components
+
+### textToSSML (students-features)
+```ts
+textToSSML("Hello world")
+// → '<speak><mark name="w_0"/>Hello <mark name="w_1"/>world</speak>'
+```
+
+### $highlightedWordIndex (students-features)
+Effector store that computes current word index from timepoints + playback time.
+
+### ChatbotHighlightedMessage (students-web)
+```tsx
+// Splits text, highlights word at highlightedIndex
+<span className={isHighlighted ? styles.Highlighted : undefined}>
+  {word}
+</span>
+```
+
+## Data Flow
+
+```
+AudioProvider.onTimeUpdate(time)
+       ↓
+t2s.setTime(time)
+       ↓
+$highlightedWordIndex = findCurrentMark(timepoints, time) → parse index
+       ↓
+Chat.tsx: useUnit(model.t2s.$highlightedWordIndex)
+       ↓
+ChatMessage → ReceivedMessage → MessageContent
+       ↓
+MessageRenderer({ text, highlightedIndex })
+       ↓
+ChatbotHighlightedMessage renders highlighted word
+```

package/src/features/chatbot/components/Chat/Chat.tsx

@@ -9,6 +9,7 @@ import { ChatInput } from '../ChatInput/ChatInput'
 import { MessageVariant } from '../../types/model.types'
 import { ColorScheme } from '../../types/style.types'
 import { ConditionalWrapper } from '../../../../lib/components/ConditionalWrapper'
+import { HighlightConfig } from '../../types/t2s.types'
 
 const TOOLBAR_HEIGHT = 140
 const BOTTOM_OFFSET = Platform.select({
@@ -23,6 +24,8 @@ type ChatProps = {
   isInputDisabled?: boolean
   colorScheme?: ColorScheme
   MessageWrapper?: ComponentType<PropsWithChildren>
+  isHighlightEnabled?: boolean
+  highlightConfig?: HighlightConfig
 }
 
 export const Chat = ({
@@ -32,6 +35,8 @@ export const Chat = ({
   isInputDisabled,
   colorScheme = ColorScheme.Blue,
   MessageWrapper,
+  isHighlightEnabled = false,
+  highlightConfig,
 }: ChatProps) => {
   const dimensions = useWindowDimensions()
   const ref = useRef<ScrollView>(null)
@@ -44,6 +49,7 @@ export const Chat = ({
   const isTranslated = useUnit(model.translation.$isTranslated)
   const audioStatus = useUnit(model.t2s.$status)
   const errorSpottingEnabled = useUnit(model.errorSpotting.$isEnabled)
+  const highlightedWordIndex = useUnit(model.t2s.$highlightedWordIndex)
 
   const messages = useStoreMap(model.$currentChatData, ({ messages }) => messages)
   const lastMessageId = messages.at(-1)?.id
@@ -68,25 +74,28 @@ export const Chat = ({
         {isOpen && (
           <View style={[styles.gap, isInputDisabled && styles.messagesWithoutInput]}>
             {messages.map((message) => (
-              <React.Fragment key={message.id}>
-                <ConditionalWrapper withWrapper={shouldWrapMessage} Wrapper={MessageWrapper}>
-                  <ChatMessage
-                    isTranslated={isTranslated}
-                    withIcon={!message.isOwnMessage && message.id === lastMessageId}
-                    message={message}
-                    variant={message.isOwnMessage ? MessageVariant.SENT : MessageVariant.RECEIVED}
-                    colorScheme={colorScheme}
-                    withErrorSpotting={errorSpottingEnabled && !message.isOwnMessage}
-                    t2sState={{
-                      isEnabled: isTextToSpeechEnabled && !message.isOwnMessage,
-                      isActive: message.id === currentPlayingId,
-                      isLoading: isTextToSpeechPending,
-                      status: audioStatus,
-                      disabled: isTextToSpeechPending,
-                    }}
-                  />
-                </ConditionalWrapper>
-              </React.Fragment>
+              <ConditionalWrapper withWrapper={shouldWrapMessage} Wrapper={MessageWrapper}>
+                <ChatMessage
+                  key={message.id}
+                  isTranslated={isTranslated}
+                  withIcon={!message.isOwnMessage && message.id === lastMessageId}
+                  message={message}
+                  variant={message.isOwnMessage ? MessageVariant.SENT : MessageVariant.RECEIVED}
+                  colorScheme={colorScheme}
+                  withErrorSpotting={errorSpottingEnabled && !message.isOwnMessage}
+                  t2sState={{
+                    isEnabled: isTextToSpeechEnabled && !message.isOwnMessage,
+                    isActive: message.id === currentPlayingId,
+                    isLoading: isTextToSpeechPending,
+                    status: audioStatus,
+                    disabled: isTextToSpeechPending,
+                  }}
+                  highlightedWordIndex={
+                    isHighlightEnabled && message.id === currentPlayingId ? highlightedWordIndex : null
+                  }
+                  highlightConfig={highlightConfig}
+                />
+              </ConditionalWrapper>
             ))}
 
             {isHelpRequestPending && <MessageLoader colorScheme={colorScheme} />}

package/src/features/chatbot/components/ChatMessage/ChatMessage.tsx

@@ -2,7 +2,7 @@ import React, { ComponentType } from 'react'
 
 import { ReceivedMessage } from './ReceivedMessage'
 import { SentMessage } from './SentMessage'
-import { T2SState } from '../../types/t2s.types'
+import { HighlightConfig, T2SState } from '../../types/t2s.types'
 import { MessageVariant } from '../../types/model.types'
 import { ConversationMessage } from '../../types/units.types'
 import { ColorScheme } from '../../types/style.types'
@@ -15,6 +15,8 @@ type ChatMessageProps = {
   withErrorSpotting?: boolean
   withIcon?: boolean
   colorScheme?: ColorScheme
+  highlightedWordIndex?: number | null
+  highlightConfig?: HighlightConfig
 }
 
 export type VariantProps = Omit<ChatMessageProps, 'variant'>

package/src/features/chatbot/components/ChatMessage/MessageContent.tsx

@@ -2,7 +2,7 @@ import React from 'react'
 import { Platform, StyleSheet, Text, View } from 'react-native'
 
 import { ConversationMessage } from '../../types/units.types'
-import { T2SState } from '../../types/t2s.types'
+import { HighlightConfig, T2SState } from '../../types/t2s.types'
 import { ColorScheme } from '../../types/style.types'
 
 import { MessageButtonsBlock } from './MessageButtonsBlock'
@@ -16,6 +16,8 @@ type MessageContentProps = {
   variant: MessageVariant
   colorScheme?: ColorScheme
   withErrorSpotting?: boolean
+  highlightedWordIndex?: number | null
+  highlightConfig?: HighlightConfig
 }
 
 const MessageContentWeb = ({
@@ -25,10 +27,23 @@ const MessageContentWeb = ({
   variant,
   colorScheme = ColorScheme.Blue,
   withErrorSpotting,
+  highlightedWordIndex,
+  highlightConfig,
 }: MessageContentProps) => {
+  const MessageRenderer = highlightConfig?.MessageRenderer
+  // console.log('[TTS Debug] MessageContent - hasRenderer:', !!MessageRenderer, 'highlightedWordIndex:', highlightedWordIndex)
+
   return (
     <Text style={styles.messageText}>
-      {text}
+      {MessageRenderer ? (
+        <MessageRenderer
+          text={text}
+          highlightedIndex={highlightedWordIndex ?? null}
+          colorScheme={colorScheme}
+        />
+      ) : (
+        text
+      )}
       <MessageButtonsBlock
         message={message}
         t2sState={t2sState}
@@ -47,11 +62,23 @@ const MessageContentMobile = ({
   variant,
   colorScheme = ColorScheme.Blue,
   withErrorSpotting,
+  highlightedWordIndex,
+  highlightConfig,
 }: MessageContentProps) => {
+  const MessageRenderer = highlightConfig?.MessageRenderer
+
   return (
     <>
       <Text style={[styles.messageText, styles.mobileMessageText]}>
-        {text}
+        {MessageRenderer ? (
+          <MessageRenderer
+            text={text}
+            highlightedIndex={highlightedWordIndex ?? null}
+            colorScheme={colorScheme}
+          />
+        ) : (
+          text
+        )}
         <View style={{ width: getSpacerSize(message, t2sState) }} />
       </Text>
       <MessageButtonsBlock

package/src/features/chatbot/components/ChatMessage/ReceivedMessage.tsx

@@ -14,6 +14,8 @@ export const ReceivedMessage = ({
   withIcon,
   colorScheme = ColorScheme.Blue,
   withErrorSpotting,
+  highlightedWordIndex,
+  highlightConfig,
 }: VariantProps) => {
   const text = isTranslated ? message.translatedMessage : message.message
 
@@ -33,6 +35,8 @@ export const ReceivedMessage = ({
           text={text ?? ''}
           colorScheme={colorScheme}
           withErrorSpotting={withErrorSpotting}
+          highlightedWordIndex={highlightedWordIndex}
+          highlightConfig={highlightConfig}
         />
       </View>
     </View>

package/src/features/chatbot/components/Chatbot.tsx

@@ -7,6 +7,7 @@ import { ChatControls } from './ChatControls/ChatControls'
 import { AUDIO_CONFIG, DEFAULT_VOICE } from '../constants'
 import { ChatBotModelContext } from '../context/ChatBotModelContext'
 import { ColorScheme } from '../types/style.types'
+import { HighlightConfig } from '../types/t2s.types'
 
 type ChatbotProps = {
   model: ChatbotModel
@@ -20,6 +21,8 @@ type ChatbotProps = {
   iconSize?: { width: number; height: number } // TODO: use number instead
   isInputDisabled?: boolean
   colorScheme?: ColorScheme
+  isHighlightEnabled?: boolean
+  highlightConfig?: HighlightConfig
 }
 
 export const Chatbot = ({
@@ -31,6 +34,8 @@ export const Chatbot = ({
   ButtonWrapper,
   iconSize,
   colorScheme = ColorScheme.Blue,
+  isHighlightEnabled = false,
+  highlightConfig,
 }: ChatbotProps) => {
   const isChatAllowed = useUnit(model.$chatAccess)
   const isAvailable = useStoreMap(model.$currentChatData, ({ isAvailable }) => isAvailable)
@@ -60,6 +65,8 @@ export const Chatbot = ({
         isTextToSpeechEnabled={isTextToSpeechEnabled}
         isInputDisabled={isInputDisabled}
         MessageWrapper={ButtonWrapper}
+        isHighlightEnabled={isHighlightEnabled}
+        highlightConfig={highlightConfig}
       />
 
       <ChatControls

package/src/features/chatbot/helpers/textToSSML.ts

@@ -0,0 +1,15 @@
+/**
+ * Converts plain text to SSML with word marks for TTS highlighting.
+ * "Hello world" -> '<speak><mark name="w_0"/>Hello <mark name="w_1"/>world</speak>'
+ */
+export const textToSSML = (text: string): string => {
+  const words = text.split(/\s+/).filter(Boolean)
+  if (words.length === 0) return '<speak></speak>'
+
+  const ssmlParts = words.map((word, i) => `<mark name="w_${i}"/>${word}`)
+  return `<speak>${ssmlParts.join(' ')}</speak>`
+}
+
+export const getWordCount = (text: string): number => {
+  return text.split(/\s+/).filter(Boolean).length
+}

package/src/features/chatbot/index.ts

@@ -1,6 +1,6 @@
 export { Chatbot } from './components/Chatbot'
 export { ChatbotModel } from './model/ChatBotModel'
-export { AudioStatus, type AudioProvider } from './types/t2s.types'
+export { AudioStatus, type AudioProvider, type HighlightConfig } from './types/t2s.types'
 export {
   MessageVariant,
   type ChatbotModelProps,

package/src/features/chatbot/model/ChatBotModel.ts

@@ -86,7 +86,6 @@ export class ChatbotModel {
     this.api = new ChatbotAPI({ ...api, postMessage: this.threadManager.postMessage })
 
     this.$chatAccess = chatAccess
-    if ($key) this.bindKey($key)
 
     this.threadManager.setApi({ getHint: api.requestHint, sendMessage: api.sendMessage })
     if (threadMessagesLimit) {
@@ -98,6 +97,7 @@ export class ChatbotModel {
       .on(this.setAlternatives, (_, payload) => transformAlternatives(payload))
       .reset(this.resetAlternatives)
 
+    this.t2s = new ChatTextToSpeechModel({ api: this.api })
     this.translation = new ChatbotTranslation({
       $context: this.$context,
       defaultMessages: this.defaultMessages,
@@ -110,16 +110,12 @@ export class ChatbotModel {
       chats: this.cache,
     })
 
-    this.t2s = new ChatTextToSpeechModel({
-      api: this.api,
-      $currentChatData: this.$currentChatData,
-      $isTranslated: this.translation.$isTranslated,
-    })
-
     this.errorSpotting = new ErrorSpottingModel({
       $currentChatData: this.$currentChatData,
     })
 
+    if ($key) this.bindKey($key)
+
     sample({
       source: this.$isOpen.updates,
       filter: (isOpen) => !isOpen,

package/src/features/chatbot/model/t2s.ts

@@ -1,18 +1,29 @@
-import { restore, createEvent, createEffect, Store } from 'effector'
+import { restore, createEvent, createEffect, combine } from 'effector'
 import { AUDIO_CONFIG, DEFAULT_VOICE } from '../constants'
 import { AudioProvider, AudioStatus, PlayTextToSpeechProps } from '../types/t2s.types'
 import { ChatbotAPI } from './ChatbotApi'
-import { ChatItem } from '../types/units.types'
+import { Timepoint } from '../types/api.types'
+import { textToSSML } from '../helpers/textToSSML'
+
+const WORD_MARK_REGEX = /^w_(\d+)$/
+
+const findCurrentMark = (timepoints: Timepoint[], time: number): string | undefined => {
+  for (let i = timepoints.length - 1; i >= 0; i--) {
+    if (timepoints[i].timeSeconds <= time) {
+      return timepoints[i].markName
+    }
+  }
+  return undefined
+}
 
 type ChatTextToSpeechModelProps = {
   api: ChatbotAPI
-  $currentChatData: Store<ChatItem>
-  $isTranslated: Store<boolean>
 }
 
 export type T2SConfig = {
   audioProvider?: AudioProvider
   textSanitizer?: (speech: string) => string
+  isHighlightEnabled?: boolean
 }
 
 type AudioConfig = {
@@ -20,34 +31,70 @@ type AudioConfig = {
   audioConfig?: typeof AUDIO_CONFIG
 }
 
+type HighlightCacheEntry = {
+  audioPath: string
+  timepoints: Timepoint[]
+}
+
 export class ChatTextToSpeechModel {
   private readonly api: ChatbotAPI
   private readonly audioCache: Record<string, string> = {}
-  private readonly $currentChatData
-  private readonly $isTranslated
+  private readonly highlightCache: Record<string, HighlightCacheEntry> = {}
   private audioProvider: AudioProvider | null = null
+  private isHighlightEnabled = false
+  private lastHighlightedIndex: number | null = null
 
   readonly setCurrentPlayingId = createEvent<string | null>()
   readonly $currentPlayingId = restore(this.setCurrentPlayingId, null)
   readonly setStatus = createEvent<AudioStatus>()
   readonly $status = restore(this.setStatus, AudioStatus.STOPPED)
 
+  readonly setTimepoints = createEvent<Timepoint[]>()
+  readonly $timepoints = restore(this.setTimepoints, [])
+
+  readonly setTime = createEvent<number>()
+  readonly $time = restore(this.setTime, 0)
+
+  readonly $highlightedWordIndex = combine(
+    this.$timepoints,
+    this.$time,
+    this.$status,
+    (timepoints, time, status) => {
+      if (!timepoints.length || status !== AudioStatus.PLAYING) {
+        return null
+      }
+
+      const currentMark = findCurrentMark(timepoints, time)
+      if (!currentMark) return null
+
+      const match = currentMark.match(WORD_MARK_REGEX)
+      const index = match ? parseInt(match[1], 10) : null
+      // console.log('[TTS Debug] Computed highlightedWordIndex:', index, '(time:', time, ', mark:', currentMark, ')')
+      return index
+    },
+  )
+
+  readonly $highlightState = combine({
+    highlightedWordIndex: this.$highlightedWordIndex,
+    playingId: this.$currentPlayingId,
+    status: this.$status,
+  })
+
   private audioConfig: AudioConfig | null = null
   private textSanitizer = (value: string) => value
 
-  constructor({ api, $currentChatData, $isTranslated }: ChatTextToSpeechModelProps) {
+  constructor({ api }: ChatTextToSpeechModelProps) {
     this.api = api
-    this.$currentChatData = $currentChatData
-    this.$isTranslated = $isTranslated
   }
 
   public setAudioConfig(config: AudioConfig | null) {
     this.audioConfig = config
   }
 
-  public setConfig({ audioProvider, textSanitizer }: T2SConfig) {
+  public setConfig({ audioProvider, textSanitizer, isHighlightEnabled }: T2SConfig) {
     if (audioProvider) this.audioProvider = audioProvider
     if (textSanitizer) this.textSanitizer = textSanitizer
+    if (isHighlightEnabled !== undefined) this.isHighlightEnabled = isHighlightEnabled
   }
 
   private getCacheKey = (messageId: string, isTranslated: boolean) => {
@@ -81,35 +128,70 @@ export class ChatTextToSpeechModel {
 
       if (messageId && !isCurrentMessage) {
         this.audioProvider.stop()
+        this.resetHighlight()
       }
 
       const cacheKey = this.getCacheKey(messageId, isTranslated)
-      let audioPath = this.audioCache[cacheKey]
+      const useHighlighting = this.isHighlightEnabled && !!this.audioProvider.onTimeUpdate
+      // console.log('[TTS Debug] useHighlighting:', useHighlighting, '(isHighlightEnabled:', this.isHighlightEnabled, ', hasOnTimeUpdate:', !!this.audioProvider.onTimeUpdate, ')')
+
+      let audioPath: string
+      let timepoints: Timepoint[] = []
+
       this.setCurrentPlayingId(messageId)
 
-      if (!audioPath) {
-        const config = this.audioConfig
-        const response = await this.api.textToSpeechTextFx({
-          input: { text },
-          voice: config?.voice ?? DEFAULT_VOICE,
-          audioConfig: config?.audioConfig ?? AUDIO_CONFIG,
-          enableTimePointing: [],
-        })
-
-        // Response might be different type based on provided AUDIO_CONFIG
-        if (typeof response.audioContent === 'string') {
-          audioPath = `data:audio/mpeg;base64,${response.audioContent}`
+      if (useHighlighting) {
+        const cached = this.highlightCache[cacheKey]
+        if (cached) {
+          audioPath = cached.audioPath
+          timepoints = cached.timepoints
         } else {
-          audioPath = response?.audioContent ?? ''
+          const config = this.audioConfig
+          const ssml = textToSSML(text)
+          const response = await this.api.textToSpeechTextFx({
+            input: { ssml },
+            voice: config?.voice ?? DEFAULT_VOICE,
+            audioConfig: config?.audioConfig ?? AUDIO_CONFIG,
+            enableTimePointing: ['SSML_MARK'],
+          })
+
+          if (typeof response.audioContent === 'string') {
+            audioPath = `data:audio/mpeg;base64,${response.audioContent}`
+          } else {
+            audioPath = response?.audioContent ?? ''
+          }
+
+          timepoints = response.timepoints ?? []
+          this.highlightCache[cacheKey] = { audioPath, timepoints }
         }
 
-        this.audioCache[cacheKey] = audioPath
+        this.setTimepoints(timepoints)
+        // console.log('[TTS Debug] Timepoints set:', timepoints.length, timepoints)
+        this.setupTimeTracking()
+      } else {
+        audioPath = this.audioCache[cacheKey]
+        if (!audioPath) {
+          const config = this.audioConfig
+          const response = await this.api.textToSpeechTextFx({
+            input: { text },
+            voice: config?.voice ?? DEFAULT_VOICE,
+            audioConfig: config?.audioConfig ?? AUDIO_CONFIG,
+            enableTimePointing: [],
+          })
+
+          if (typeof response.audioContent === 'string') {
+            audioPath = `data:audio/mpeg;base64,${response.audioContent}`
+          } else {
+            audioPath = response?.audioContent ?? ''
+          }
+
+          this.audioCache[cacheKey] = audioPath
+        }
       }
 
       this.setStatus(AudioStatus.PLAYING)
       await this.audioProvider.play(audioPath, () => {
-        this.setStatus(AudioStatus.STOPPED)
-        this.setCurrentPlayingId(null)
+        this.onPlaybackComplete()
       })
     } catch (error) {
       this.reset()
@@ -117,12 +199,43 @@ export class ChatTextToSpeechModel {
     }
   }
 
-  public async playLatest() {
-    const lastMessage = this.$currentChatData.getState().messages.at(-1)
-    if (!lastMessage) return
+  private setupTimeTracking() {
+    if (!this.audioProvider?.onTimeUpdate) {
+      // console.log('[TTS Debug] No onTimeUpdate on audioProvider!')
+      return
+    }
+
+    // console.log('[TTS Debug] Setting up time tracking')
+    this.lastHighlightedIndex = null
+
+    this.audioProvider.onTimeUpdate((time: number) => {
+      const timepoints = this.$timepoints.getState()
+      const currentMark = findCurrentMark(timepoints, time)
+      if (!currentMark) return
+
+      const match = currentMark.match(WORD_MARK_REGEX)
+      const newIndex = match ? parseInt(match[1], 10) : null
+
+      // Only update if word index changed
+      if (newIndex !== this.lastHighlightedIndex) {
+        // console.log('[TTS Debug] Word changed:', this.lastHighlightedIndex, '→', newIndex)
+        this.lastHighlightedIndex = newIndex
+        this.setTime(time)
+      }
+    })
+  }
+
+  private onPlaybackComplete = () => {
+    this.setStatus(AudioStatus.STOPPED)
+    this.setCurrentPlayingId(null)
+    this.resetHighlight()
+    this.audioProvider?.onTimeUpdate?.(null)
+  }
 
-    const isTranslated = this.$isTranslated.getState()
-    return this.play({ message: lastMessage, isTranslated })
+  private resetHighlight() {
+    this.setTimepoints([])
+    this.setTime(0)
+    this.lastHighlightedIndex = null
   }
 
   public stop() {
@@ -134,5 +247,7 @@ export class ChatTextToSpeechModel {
     this.stop()
     this.setStatus(AudioStatus.STOPPED)
     this.setCurrentPlayingId(null)
+    this.resetHighlight()
+    this.audioProvider?.onTimeUpdate?.(null)
   })
 }

package/src/features/chatbot/types/api.types.ts

@@ -71,7 +71,8 @@ export type GoogleTranslateResponse = {
 
 export type GoogleText2SpeechPayload = {
   input: {
-    text: string
+    text?: string
+    ssml?: string
   }
   voice: {
     languageCode: string

package/src/features/chatbot/types/t2s.types.ts

@@ -1,3 +1,4 @@
+import React from 'react'
 import { ConversationMessage } from './units.types'
 
 export enum AudioStatus {
@@ -11,6 +12,12 @@ export type AudioProvider = {
   pause: () => void
   resume: () => void
   stop: () => void
+  onTimeUpdate?: (callback: ((time: number) => void) | null) => void
+  getCurrentTime?: () => number
+}
+
+export type HighlightConfig = {
+  MessageRenderer?: React.ComponentType<{ text: string; highlightedIndex: number | null; colorScheme?: string }>
 }
 
 export type PlayTextToSpeechProps = {

package/src/index.ts

@@ -9,5 +9,4 @@ export * from './features/problemSelector'
 export * from './features/gifCelebrations'
 export * from './features/exampleSolution'
 export * from './features/keyboard'
-export * from './features/openEnded'
 export * from './shared/icons'