@magmamath/students-features 0.10.0 → 0.11.0-rc.0

package/src/features/chatbot/model/ChatBotModel.ts

@@ -7,7 +7,7 @@ import {
   restore,
   sample,
   type Store,
-  StoreWritable
+  StoreWritable,
 } from 'effector'
 import {
   ChatbotContext,
@@ -23,7 +23,7 @@ import {
 import { GetAlternativesResponse } from '../types/api.types'
 import { ChatbotAPI } from './ChatbotApi'
 import { ChatbotTranslation } from './ChatbotTranslation'
-import { ChatTextToSpeechModel, T2SConfig } from './t2s'
+import { ChatTTSModel } from './ChatTTSModel'
 import { messagesHelper } from './messagesHelper'
 import { ChatBotThreadManager } from './ChatBotThreadManager'
 import { ChatsCacheModel, defaultChatData } from './ChatsCacheModel'
@@ -34,6 +34,7 @@ import { ChatBotTextManager } from './ChatBoxTextManager'
 import { ControlledSample } from '../../../lib/effector/ControlledSample'
 import { transformAlternatives } from '../helpers'
 import { IS_WEB } from '@magmamath/react-native-ui'
+import { T2SConfig } from '../types/t2s.types'
 
 export class ChatbotModel {
   public readonly textManager = new ChatBotTextManager()
@@ -45,7 +46,7 @@ export class ChatbotModel {
   public readonly api: ChatbotAPI
 
   public readonly translation: ChatbotTranslation
-  public readonly t2s: ChatTextToSpeechModel
+  public readonly t2s: ChatTTSModel
   public readonly errorSpotting: ErrorSpottingModel
   public readonly textInput = new ChatBotTextInput()
 
@@ -57,7 +58,7 @@ export class ChatbotModel {
 
   readonly resetAlternatives = createEvent()
   readonly setAlternatives = createEvent<GetAlternativesResponse | null>()
-  readonly $alternatives: Store<GetAlternativesResponse | null>
+  readonly $alternatives = createStore<GetAlternativesResponse | null>(null)
 
   readonly setIsOpen = createEvent<boolean>()
   readonly toggleOpen = createEvent()
@@ -86,18 +87,18 @@ export class ChatbotModel {
     this.api = new ChatbotAPI({ ...api, postMessage: this.threadManager.postMessage })
 
     this.$chatAccess = chatAccess
+    if ($key) this.bindKey($key)
 
     this.threadManager.setApi({ getHint: api.requestHint, sendMessage: api.sendMessage })
     if (threadMessagesLimit) {
       this.threadManager.setMessagesLimit(threadMessagesLimit)
     }
 
-    this.$alternatives = createStore<GetAlternativesResponse | null>(null)
+    this.$alternatives
       .on(this.api.getAlternativesFx.doneData, (_, payload) => transformAlternatives(payload))
       .on(this.setAlternatives, (_, payload) => transformAlternatives(payload))
       .reset(this.resetAlternatives)
 
-    this.t2s = new ChatTextToSpeechModel({ api: this.api })
     this.translation = new ChatbotTranslation({
       $context: this.$context,
       defaultMessages: this.defaultMessages,
@@ -110,11 +111,15 @@ export class ChatbotModel {
       chats: this.cache,
     })
 
-    this.errorSpotting = new ErrorSpottingModel({
+    this.t2s = new ChatTTSModel({
+      api: this.api,
       $currentChatData: this.$currentChatData,
+      $isTranslated: this.translation.$isTranslated,
     })
 
-    if ($key) this.bindKey($key)
+    this.errorSpotting = new ErrorSpottingModel({
+      $currentChatData: this.$currentChatData,
+    })
 
     sample({
       source: this.$isOpen.updates,
@@ -122,6 +127,14 @@ export class ChatbotModel {
       target: [this.t2s.reset, this.errorSpotting.spottingHidden],
     })
 
+    sample({
+      clock: this.setIsOpen,
+      source: this.$currentChatData,
+      filter: ({ hasBeenOpened }, isOpen) => isOpen && !hasBeenOpened,
+      fn: ({ key }) => ({ key, data: { hasBeenOpened: true } }),
+      target: this.cache.update,
+    })
+
     sample({
       clock: this.removeLastMessage,
       source: [this.$lastMessageKey, this.cache.$data] as const,
@@ -156,7 +169,12 @@ export class ChatbotModel {
         ...source,
       }),
       effect: createEffect(
-        ({ currentChatData, initOnly, chatAccess }: StartConversationFxData) => {
+        ({
+          currentChatData,
+          initOnly,
+          chatAccess,
+          withGreetingMessage = true,
+        }: StartConversationFxData) => {
           const { key, messages } = currentChatData
 
           if (!chatAccess) return
@@ -166,7 +184,7 @@ export class ChatbotModel {
             return this.reinitConversation({ key })
           }
           this.cache.add(key, {
-            messages: [this.defaultMessages.start],
+            messages: withGreetingMessage ? [this.defaultMessages.start] : [],
             isAvailable: true,
             currentThread: null,
           })
@@ -186,29 +204,58 @@ export class ChatbotModel {
 
   private createSendHelpRequestFx() {
     const effect = attach({
-      source: { key: this.$currentKey, context: this.$context, cache: this.cache.$data },
+      source: {
+        key: this.$currentKey,
+        context: this.$context,
+        cache: this.cache.$data,
+        alternatives: this.$alternatives,
+      },
       mapParams: (params: SendHelpParams, source) => ({
         ...params,
         ...source,
       }),
       effect: createEffect(
-        async ({ message, translatedMessage, key, context, isFreeText, cache }: SendHelpProps) => {
+        async ({
+          message,
+          translatedMessage,
+          key,
+          context,
+          isFreeText,
+          skipOwnMessage,
+          cache,
+          alternatives,
+        }: SendHelpProps) => {
           if (!context) throw new Error('Trying to send message in chat without context')
           this.setLastMessageKey(key)
-          this.cache.addMessages(key, {
-            messages: [messagesHelper.createOwn({ message, translatedMessage })],
-          })
+          if (skipOwnMessage) {
+            this.cache.clearMessages(key)
+          } else {
+            this.cache.addMessages(key, {
+              messages: [messagesHelper.createOwn({ message, translatedMessage })],
+            })
+          }
+
+          const chatHistory =
+            context.assignmentId && context.problemId
+              ? {
+                  exerciseId: context.assignmentId,
+                  problemId: context.problemId,
+                  chatVersion: alternatives?.version,
+                }
+              : undefined
 
           // Request is overridden by ThreadManager.postMessage (delegates to sendMessage/getHint)
           const response = await this.api.postMessageFx({
             answer: context.answer,
             answerOptions: context.answerOptions,
-            problemDescription: context.problem,
+            problemDescription: context.problemDescription,
+            problemStandards: context.problemStandards,
             problemAnswer: context.problemAnswer,
             language: context.language,
             imageAltText: context.imageDescription,
             imageCdnId: context.imageCdnId,
             conversationId: cache[key].conversationId,
+            chatHistory,
             ...(isFreeText && { freeTextMessage: message }),
           })
 
@@ -294,6 +341,10 @@ export class ChatbotModel {
     this.cache.setSingleMessage(key, { ...message, options })
   }
 
+  public addMessage({ key, ...message }: SetHintMessageProps) {
+    this.cache.addMessages(key, { messages: [message] })
+  }
+
   public readonly reinitConversation = ({ key }: ReinitConversationProps) => {
     this.cache.reinitChatThread(key, { initialOption: this.defaultMessages.anotherHintOption })
     this.threadManager.reset(key)

package/src/features/chatbot/model/ChatBotThreadManager.ts

@@ -10,10 +10,17 @@ import {
 import { ChatbotMessages, ChatRequestType, IChatDrawBoardAdapter } from '../types/model.types'
 import { CHAT_BOT_MODEL_DEFAULTS } from './constants'
 import { prepareChatRequestPayload } from '../helpers'
+import { RequestOptionalConfig } from 'src/lib/effector/createControllerEffect'
 
 type ThreadManagerApi = {
-  getHint: (props: ChatHintPayload) => Promise<ChatHintResponse>
-  sendMessage: (props: SendMessagePayload) => Promise<SendMessageResponse>
+  getHint: (
+    props: ChatHintPayload,
+    requestConfig?: RequestOptionalConfig,
+  ) => Promise<ChatHintResponse>
+  sendMessage: (
+    props: SendMessagePayload,
+    requestConfig?: RequestOptionalConfig,
+  ) => Promise<SendMessageResponse>
 }
 
 type ChatBotThreadManagerProps = {
@@ -69,6 +76,7 @@ export class ChatBotThreadManager {
 
   public readonly postMessage = async (
     payload: PostMessagePayload,
+    requestConfig?: RequestOptionalConfig,
   ): Promise<PostMessageResponse> => {
     if (!this.api) {
       throw new Error(
@@ -80,8 +88,8 @@ export class ChatBotThreadManager {
 
     const result: SendMessageResponse | ChatHintResponse =
       type === ChatRequestType.FREETEXT
-        ? await this.api.sendMessage(data)
-        : await this.api.getHint(data)
+        ? await this.api.sendMessage(data, requestConfig)
+        : await this.api.getHint(data, requestConfig)
 
     threadItem.history.push(result.message)
 

package/src/features/chatbot/model/ChatTTSHighlightModel.ts

@@ -0,0 +1,64 @@
+import { restore, createEvent, combine } from 'effector'
+import { Timepoint } from '../types/api.types'
+import { OnTimeUpdate } from '../types/t2s.types'
+import { findCurrentMark } from '../helpers/ttsHighlight'
+import { WORD_MARK_REGEX } from '../constants'
+
+/**
+ * Manages word-by-word highlight state during TTS playback.
+ * Tracks which word index should be highlighted based on audio timepoints
+ * and current playback time from the AudioProvider.
+ */
+export class ChatTTSHighlightModel {
+  private lastMark: string | undefined = undefined
+
+  readonly setTimepoints = createEvent<Timepoint[]>()
+  readonly setTime = createEvent<number>()
+  readonly setIsActive = createEvent<boolean>()
+  readonly reset = createEvent()
+
+  readonly $time = restore(this.setTime, 0).reset(this.reset)
+  readonly $timepoints = restore(this.setTimepoints, []).reset(this.reset)
+  readonly $isActive = restore(this.setIsActive, false).reset(this.reset)
+
+  readonly $highlightedWordIndex = combine(
+    this.$timepoints,
+    this.$time,
+    this.$isActive,
+    (timepoints, time, isActive) => {
+      if (!isActive || !timepoints.length) return -1
+
+      const currentMark = findCurrentMark(timepoints, time)
+      if (!currentMark) return -1
+
+      const match = currentMark.match(WORD_MARK_REGEX)
+      return match ? parseInt(match[1], 10) : -1
+    },
+  )
+
+  /**
+   * Sets timepoints and starts listening to AudioProvider time updates.
+   * Only triggers store updates when the highlighted word actually changes.
+   */
+  activate(timepoints: Timepoint[], onTimeUpdate: OnTimeUpdate) {
+    this.setTimepoints(timepoints)
+    this.setIsActive(true)
+    this.lastMark = undefined
+
+    onTimeUpdate((time: number) => {
+      const currentMark = findCurrentMark(this.$timepoints.getState(), time)
+      if (!currentMark) return
+
+      if (currentMark !== this.lastMark) {
+        this.lastMark = currentMark
+        this.setTime(time)
+      }
+    })
+  }
+
+  deactivate(onTimeUpdate: OnTimeUpdate | null) {
+    onTimeUpdate?.(null)
+    this.lastMark = undefined
+    this.reset()
+  }
+}

package/src/features/chatbot/model/ChatTTSModel.ts

@@ -0,0 +1,168 @@
+import { restore, createEvent, createEffect, sample, Store } from 'effector'
+import { AUDIO_CONFIG, DEFAULT_VOICE } from '../constants'
+import {
+  AudioCache,
+  AudioConfig,
+  AudioProvider,
+  AudioStatus,
+  PlayTextToSpeechProps,
+  T2SConfig,
+} from '../types/t2s.types'
+
+import { ChatItem } from '../types/units.types'
+import { textToSSML } from '../helpers/ttsHighlight'
+import { ChatbotAPI } from './ChatbotApi'
+import { ChatTTSHighlightModel } from './ChatTTSHighlightModel'
+
+type ChatTextToSpeechModelProps = {
+  api: ChatbotAPI
+  $currentChatData: Store<ChatItem>
+  $isTranslated: Store<boolean>
+}
+
+export class ChatTTSModel {
+  private readonly api: ChatbotAPI
+  public readonly wordHighlight = new ChatTTSHighlightModel()
+  private readonly audioCache = new Map<string, AudioCache>()
+  private readonly $currentChatData
+  private readonly $isTranslated
+  private audioProvider: AudioProvider | null = null
+  private wordHighlightEnabled = false
+
+  readonly setCurrentPlayingId = createEvent<string | null>()
+  readonly $currentPlayingId = restore(this.setCurrentPlayingId, null)
+  readonly setStatus = createEvent<AudioStatus>()
+  readonly $status = restore(this.setStatus, AudioStatus.STOPPED)
+
+  private audioConfig: AudioConfig | null = null
+  private textSanitizer = (value: string) => value
+
+  constructor({ api, $currentChatData, $isTranslated }: ChatTextToSpeechModelProps) {
+    this.api = api
+    this.$currentChatData = $currentChatData
+    this.$isTranslated = $isTranslated
+
+    sample({
+      clock: $isTranslated,
+      target: this.reset,
+    })
+  }
+
+  public setAudioConfig(config: AudioConfig | null) {
+    this.audioConfig = config
+  }
+
+  public setConfig({ audioProvider, textSanitizer, isWordHighlightEnabled }: T2SConfig) {
+    if (audioProvider) this.audioProvider = audioProvider
+    if (textSanitizer) this.textSanitizer = textSanitizer
+    // Highlighting requires audioProvider.onTimeUpdate to sync playback time with word highlights
+    this.wordHighlightEnabled = !!isWordHighlightEnabled && !!this.audioProvider?.onTimeUpdate
+  }
+
+  public get isWordHighlightEnabled() {
+    return this.wordHighlightEnabled
+  }
+
+  private getCacheKey = (messageId: string, isTranslated: boolean) => {
+    return `${messageId}-${isTranslated ? 'translated' : 'original'}`
+  }
+
+  private async fetchAudio(text: string): Promise<AudioCache> {
+    const config = this.audioConfig
+    const response = await this.api.textToSpeechTextFx({
+      input: this.wordHighlightEnabled ? { ssml: textToSSML(text) } : { text },
+      voice: config?.voice ?? DEFAULT_VOICE,
+      audioConfig: config?.audioConfig ?? AUDIO_CONFIG,
+      enableTimePointing: this.wordHighlightEnabled ? ['SSML_MARK'] : [],
+    })
+
+    const audioPath =
+      typeof response.audioContent === 'string'
+        ? `data:audio/mpeg;base64,${response.audioContent}`
+        : (response?.audioContent ?? '')
+
+    return { audioPath, timepoints: response.timepoints ?? [] }
+  }
+
+  public async play({ message, isTranslated }: PlayTextToSpeechProps) {
+    if (!this.audioProvider) throw new Error('Audio provider is not set. Use setConfig.')
+    const messageText = isTranslated ? message.translatedMessage! : message.message
+    const messageId = message.id
+    const text = this.textSanitizer(messageText)
+
+    try {
+      const status = this.$status.getState()
+      const playingId = this.$currentPlayingId.getState()
+      const isPlaying = status === AudioStatus.PLAYING
+      const isPaused = status === AudioStatus.PAUSED
+      const isCurrentMessage = messageId === playingId
+
+      if (isPlaying && isCurrentMessage) {
+        this.audioProvider.pause()
+        if (this.wordHighlightEnabled) this.wordHighlight.setIsActive(false)
+        this.setStatus(AudioStatus.PAUSED)
+        return
+      }
+
+      if (isPaused && isCurrentMessage) {
+        this.audioProvider.resume()
+        if (this.wordHighlightEnabled) this.wordHighlight.setIsActive(true)
+        this.setStatus(AudioStatus.PLAYING)
+        return
+      }
+
+      if (messageId && !isCurrentMessage) {
+        this.audioProvider.stop()
+        this.wordHighlight.deactivate(this.audioProvider?.onTimeUpdate ?? null)
+      }
+
+      const cacheKey = this.getCacheKey(messageId, isTranslated)
+      this.setCurrentPlayingId(messageId)
+
+      const cached = this.audioCache.get(cacheKey)
+      const { audioPath, timepoints } = cached ?? (await this.fetchAudio(text))
+
+      if (!cached) {
+        this.audioCache.set(cacheKey, { audioPath, timepoints })
+      }
+
+      if (this.wordHighlightEnabled && this.audioProvider?.onTimeUpdate && timepoints.length > 0) {
+        this.wordHighlight.activate(timepoints, this.audioProvider.onTimeUpdate)
+      }
+
+      this.setStatus(AudioStatus.PLAYING)
+      await this.audioProvider.play(audioPath, () => {
+        this.onPlaybackComplete()
+      })
+    } catch (error) {
+      void this.reset()
+      console.error('Error playing text-to-speech: ', error)
+    }
+  }
+
+  public async playLatest() {
+    const lastMessage = this.$currentChatData.getState().messages.at(-1)
+    if (!lastMessage) return
+
+    const isTranslated = this.$isTranslated.getState()
+    return this.play({ message: lastMessage, isTranslated })
+  }
+
+  private onPlaybackComplete = () => {
+    this.setStatus(AudioStatus.STOPPED)
+    this.setCurrentPlayingId(null)
+    this.wordHighlight.deactivate(this.audioProvider?.onTimeUpdate ?? null)
+  }
+
+  public stop() {
+    if (!this.audioProvider) throw new Error('Audio provider is not set. Use setAudioProvider.')
+    this.audioProvider.stop()
+  }
+
+  public readonly reset = createEffect(() => {
+    this.stop()
+    this.setStatus(AudioStatus.STOPPED)
+    this.setCurrentPlayingId(null)
+    this.wordHighlight.deactivate(this.audioProvider?.onTimeUpdate ?? null)
+  })
+}

package/src/features/chatbot/model/ChatbotApi.ts

@@ -13,6 +13,7 @@ export class ChatbotAPI {
   constructor(api: ChatbotApiRequests) {
     this.postMessageFx = createControllerEffect(api.postMessage)
     this.getAlternativesFx = createEffect(api.getAlternatives)
+    this.textToSpeechTextFx = createEffect(api.postTextToSpeech)
     this.translateTextFx = createEffect(
       async (payload: GoogleTranslatePayload): Promise<GoogleTranslateResponse> => {
         const response = await api.postTranslateText({
@@ -32,6 +33,5 @@ export class ChatbotAPI {
         }
       },
     )
-    this.textToSpeechTextFx = createEffect(api.postTextToSpeech)
   }
 }

package/src/features/chatbot/model/ChatsCacheModel.ts

@@ -17,6 +17,10 @@ type AddMessagesProps = {
   thread?: ThreadItem | null
 }
 
+type AddMessageProps = {
+  message: ConversationMessage
+}
+
 export function defaultChatData(props: Partial<ChatItem> = {}): ChatItem {
   return {
     messages: [],
@@ -67,6 +71,13 @@ export class ChatsCacheModel {
     })
   }
 
+  public clearMessages(key: string) {
+    this.cache.setField({
+      key,
+      fn: (current) => (current ? { ...current, messages: [] } : undefined),
+    })
+  }
+
   public setSingleMessage(key: string, message: ConversationMessage) {
     this.cache.setField({
       key,

package/src/features/chatbot/types/api.types.ts

@@ -1,4 +1,4 @@
-import { ChatAnswer, FullStrokeData, Stroke } from './units.types'
+import { ChatAnswer, Stroke } from './units.types'
 
 export type GetAlternativesPayload = {
   municipalityId: string
@@ -10,21 +10,36 @@ export type AlternativesOptions = {
   hints: ChatAnswer[]
 }
 
+export type ChatHistory = { exerciseId: string; problemId: string; chatVersion?: string }
+
+export type ProblemStandard = {
+  id: string
+  name: string
+  description: string
+  numberOfMatchedSkills: number
+}
+
 export type GetAlternativesResponse = {
+  /**
+   * @deprecated chatAccess depends on assignment.lavaChat or class grade
+   */
   chatAccess?: boolean
   isOpenChatEnabled?: boolean
+  municipalityChatEnable: boolean
   version: string
   options: AlternativesOptions
 }
 
 export type SendMessagePayload = {
-  problemDescription?: string
-  correctAnswer?: string | string[]
-  answerOptions?: string[]
-  altText?: string
-  language?: string
   message: string
+  altText?: string
+  answerOptions?: string[]
+  chatHistory?: ChatHistory
+  correctAnswer?: string | string[]
   conversationId?: string
+  problemDescription?: string
+  problemStandards?: ProblemStandard[]
+  language?: string
 }
 
 export type SendMessageResponse = {
@@ -36,12 +51,14 @@ export type PostMessagePayload = {
   language: string
   problemAnswer: string[]
   problemDescription: string
+  problemStandards?: ProblemStandard[]
   answer?: string | string[]
   answerOptions?: string[]
   imageAltText?: string
   imageCdnId?: string
   freeTextMessage?: string
   conversationId?: string | null
+  chatHistory?: ChatHistory
 }
 
 export type PostMessageResponse = {
@@ -66,9 +83,7 @@ export type GoogleTranslateResponse = {
 }
 
 export type GoogleText2SpeechPayload = {
-  input: {
-    text: string
-  }
+  input: { text: string } | { ssml: string }
   voice: {
     languageCode: string
     name: string
@@ -93,6 +108,7 @@ export type GoogleText2SpeechResponse = {
 
 export type ChatHintPayload = {
   problemDescription: string
+  problemStandards?: ProblemStandard[]
   correctAnswer: string[]
   userAnswer?: string | string[]
   stroke?: Stroke[]
@@ -108,13 +124,3 @@ export type ChatHintResponse = {
   conversationId?: string | null
   errorStrokeIds?: string[]
 }
-export type ErrorSpottingPayload = {
-  strokes: FullStrokeData[]
-  problemDescription: string
-  correctAnswer: string[]
-  userAnswer: string | string[]
-  hint: string
-}
-export type ErrorSpottingResponse = {
-  highlightedStrokeIds: string[]
-}