@magmamath/students-features 0.11.0-rc.9 → 1.0.1

package/src/features/chatbot/types/model.types.ts

@@ -2,6 +2,8 @@ import type { Store, StoreWritable } from 'effector'
 import {
   ChatHintPayload,
   ChatHintResponse,
+  ErrorSpottingPayload,
+  ErrorSpottingResponse,
   GetAlternativesPayload,
   GetAlternativesResponse,
   GoogleText2SpeechPayload,
@@ -10,7 +12,6 @@ import {
   GoogleTranslateResponse,
   PostMessagePayload,
   PostMessageResponse,
-  ProblemStandard,
   SendMessagePayload,
   SendMessageResponse,
 } from './api.types'
@@ -24,7 +25,6 @@ export type ChatbotMessages = {
   start: ConversationMessage
   endMessage: ConversationMessage
   anotherHintOption: ChatAnswer
-  wantHelp: ChatAnswer
 }
 
 export type ChatbotApiRequests = {
@@ -43,6 +43,10 @@ export type ChatbotApiRequests = {
   postTextToSpeech: (
     props: WithAbortSignal<GoogleText2SpeechPayload>,
   ) => Promise<GoogleText2SpeechResponse>
+  /**
+   * @deprecated merged with hint
+   */
+  spotError?: (props: ErrorSpottingPayload) => Promise<ErrorSpottingResponse>
 }
 
 type ChatbotSettings = {
@@ -66,14 +70,12 @@ export type SendHelpParams = {
   message: string
   translatedMessage?: string
   isFreeText?: boolean
-  skipOwnMessage?: boolean
 }
 
 export type SendHelpProps = SendHelpParams & {
   key: string
   context: ChatbotContext | null
   cache: Record<string, ChatItem>
-  alternatives?: GetAlternativesResponse | null
   isFreeText?: boolean
 }
 
@@ -127,15 +129,13 @@ export type IChatDrawBoardAdapter = {
 
 export type ChatbotContext = {
   language: string
-  problemDescription: string
+  problem: string
   problemAnswer: string[]
   translationTargetLanguage: string
-  assignmentId?: string
   answer?: string | string[]
   answerOptions?: string[]
   imageDescription?: string
   imageCdnId?: string
-  problemId?: string
   problemStandards?: ProblemStandard[]
 }
 
@@ -150,7 +150,7 @@ export enum ChatRequestType {
 
 type FreeTextWithConversationId = {
   type: ChatRequestType.FREETEXT
-  data: Pick<SendMessagePayload, 'message' | 'conversationId' | 'chatHistory'>
+  data: Pick<SendMessagePayload, 'message' | 'conversationId'>
 }
 
 type FreeTextNewConversation = {
@@ -163,4 +163,11 @@ export type HintRequest = {
   data: ChatHintPayload
 }
 
+export type ProblemStandard = {
+  id: string
+  name: string
+  description: string
+  numberOfMatchedSkills: number
+}
+
 export type ChatRequestPayload = FreeTextWithConversationId | FreeTextNewConversation | HintRequest

package/src/features/chatbot/types/units.types.ts

@@ -36,5 +36,4 @@ export type ChatItem = {
   isAvailable: boolean
   currentThread: IThreadItem | null
   conversationId?: string | null
-  hasBeenOpened?: boolean
 }

package/src/features/gifCelebrations/helpers.ts

@@ -52,6 +52,7 @@ export const generateProblemsGifsTable = (
     const selectedGif = unshownGif || gifsInGroup[0]
 
     if (selectedGif) {
+      selectedGif.isShown = true
       result[problemNumber] = {
         url: selectedGif.url,
         group: groupKey,

package/src/features/voice/constants.ts

@@ -0,0 +1,15 @@
+export const VOICE_RECORDER_CONTAINER_SIZES = {
+  COLLAPSED: 61,
+  EXPANDED: 152,
+}
+export const VOICE_RECORDER_MAX_DURATION_MS = 60000 // 1 minute
+export const VOICE_RECORDER_MIN_DURATION_MS = 300
+export enum VoiceRecorderState {
+  IDLE,
+  RECORDING,
+  PAUSED,
+}
+export const DROPDOWN_MAX_CONTENT_HEIGHT = 330
+export const TRANSCRIPT_RETRY_INTERVAL_MS = 4000
+export const TRANSCRIPT_MAX_RETRIES = 15
+export const NO_AUDIO_BE_MESSAGE = 'No audio'

package/src/features/voice/helpers.ts

@@ -0,0 +1,251 @@
+import { IS_MOBILE, IS_WEB } from '@magmamath/react-native-ui'
+import {
+  AudioPlayer,
+  AudioRecorder,
+  createAudioPlayer,
+  getRecordingPermissionsAsync,
+  requestRecordingPermissionsAsync,
+} from 'expo-audio'
+import * as FileSystem from 'expo-file-system'
+import { VoicePlayerApi, VoiceRecorderApi } from './types'
+import { VOICE_RECORDER_MAX_DURATION_MS } from './constants'
+import { PlayerState } from './playing/model'
+import { AudioStatus } from '../chatbot'
+import { AxiosResponse } from 'axios'
+
+export const formatDuration = (ms: number): string => {
+  const totalSeconds = Math.floor(ms / 1000)
+  const minutes = Math.floor(totalSeconds / 60)
+  const seconds = totalSeconds % 60
+  return `${minutes}:${seconds.toString().padStart(2, '0')}`
+}
+
+export const ensureRecordingPermissions = async () => {
+  const { status } = await getRecordingPermissionsAsync()
+
+  if (status !== 'granted') {
+    const { granted } = await requestRecordingPermissionsAsync()
+    return granted
+  }
+
+  return true
+}
+
+const getAvailableInputsWeb = async () => {
+  if (!navigator.mediaDevices?.enumerateDevices) return []
+
+  const devices = await navigator.mediaDevices.enumerateDevices()
+  return devices.filter((d) => d.kind === 'audioinput')
+}
+
+export const getAvailableInputs = async (recorder: AudioRecorder) => {
+  if (IS_WEB) {
+    return await getAvailableInputsWeb()
+  }
+  if (IS_MOBILE) {
+    return await recorder.getAvailableInputs()
+  }
+  return []
+}
+
+export const addAudioInputsListenerWeb = (callback: (devices: MediaDeviceInfo[]) => void) => {
+  if (!IS_WEB || !navigator.mediaDevices?.addEventListener) {
+    return () => {}
+  }
+
+  const handleDeviceChange = async () => {
+    const devices = await getAvailableInputsWeb()
+    callback(devices)
+  }
+
+  handleDeviceChange()
+
+  navigator.mediaDevices.addEventListener('devicechange', handleDeviceChange)
+
+  return () => {
+    navigator.mediaDevices.removeEventListener('devicechange', handleDeviceChange)
+  }
+}
+/**
+ *
+ * @param fileUri
+ * @param externalPlayer - is optional expo-audio player, but should be provided
+ * because creating player is resource consuming operation
+ */
+export const getAudioDurationMs = (
+  fileUri: string,
+  externalPlayer?: AudioPlayer,
+): Promise<number | undefined> => {
+  if (!fileUri) return Promise.resolve(undefined)
+
+  const player = externalPlayer ?? createAudioPlayer()
+  const isInternalPlayer = !externalPlayer
+
+  player.replace(fileUri)
+
+  return new Promise((resolve, reject) => {
+    const subscription = player.addListener('playbackStatusUpdate', () => {
+      if (player.duration !== undefined && player.duration > 0) {
+        subscription.remove()
+
+        // player.duration is in seconds (e.g., 1.234567)
+        const durationMs = Math.round(player.duration * 1000) // convert to milliseconds
+
+        if (isInternalPlayer) {
+          player.remove()
+        }
+
+        resolve(durationMs)
+      }
+    })
+
+    setTimeout(() => {
+      subscription.remove()
+
+      if (isInternalPlayer) {
+        player.remove()
+      }
+
+      reject(new Error('Timeout loading audio metadata'))
+    }, 3000)
+  })
+}
+
+type GenerateAudioFormDataParams = {
+  audioFileUri: string
+  durationMs: number
+  fileName?: string
+}
+
+export const generateAudioFormData = async ({
+  audioFileUri,
+  durationMs,
+  fileName,
+}: GenerateAudioFormDataParams): Promise<FormData> => {
+  const formData = new FormData()
+  const duration = Math.min(durationMs, VOICE_RECORDER_MAX_DURATION_MS).toString()
+
+  if (IS_WEB) {
+    const response = await fetch(audioFileUri)
+    const blob = await response.blob()
+    const filename = fileName + blob.type.replace('audio/', '.')
+
+    formData.append('file', blob, filename)
+    formData.append('filename', filename)
+    formData.append('duration', duration)
+
+    return formData
+  }
+
+  const filename = fileName + '.m4a'
+  const file = {
+    uri: audioFileUri,
+    type: 'audio/m4a',
+    name: filename,
+  } as any
+
+  formData.append('file', file, filename)
+  formData.append('filename', filename)
+  formData.append('duration', duration)
+
+  return formData
+}
+
+export const uploadAudioFileRequestGenerator = (
+  audioFileFormData: FormData,
+  api: VoiceRecorderApi,
+) => {
+  const controller = new AbortController()
+
+  return {
+    uploadAudio: () =>
+      api.uploadAudioFile(
+        {
+          audioFileFormData,
+        },
+        //@ts-ignore
+        {
+          signal: controller.signal,
+        },
+      ),
+    controller,
+  }
+}
+
+const downloadAudioMobile = async (response: any, audioFileId: string) => {
+  const { config } = response
+  const url = config.url.startsWith('http')
+    ? config.url
+    : `${config.baseURL?.replace(/\/$/, '')}/${config.url.replace(/^\//, '')}`
+
+  const fileName = `audio-${audioFileId}.mp4`
+  const fileUri = `${FileSystem.cacheDirectory}${fileName}`
+  const authHeader = config.headers?.Authorization || config.headers?.authorization
+
+  const downloadResult = await FileSystem.downloadAsync(
+    url,
+    fileUri,
+    authHeader ? { headers: { Authorization: authHeader } } : undefined,
+  )
+
+  if (downloadResult.status !== 200) {
+    throw new Error(`Failed to download audio file. Status: ${downloadResult.status}`)
+  }
+
+  return {
+    uri: downloadResult.uri,
+    fileName,
+    fileType: 'audio/mp4',
+  }
+}
+
+const downloadAudioWeb = (response: AxiosResponse<Blob>, audioFileId: string) => {
+  const blob: Blob = response.data
+
+  if (!(blob instanceof Blob)) {
+    throw new Error('Unable to extract audio blob from response')
+  }
+
+  return {
+    uri: URL.createObjectURL(blob),
+    fileName: `audio-${audioFileId}.mp4`,
+    fileType: blob.type || 'audio/mp4',
+  }
+}
+
+export const downloadAudioFileRequestGenerator = (audioFileId: string, api: VoicePlayerApi) => {
+  const controller = new AbortController()
+
+  return {
+    downloadAudio: async () => {
+      const response = await api.downloadAudioFile(audioFileId, {
+        //@ts-ignore
+        signal: controller.signal,
+      })
+
+      return IS_WEB
+        ? downloadAudioWeb(response, audioFileId)
+        : downloadAudioMobile(response, audioFileId)
+    },
+    controller,
+  }
+}
+
+export const getActiveAttempt = (attemptOffset?: number, attemptsCount?: number): number | null => {
+  if (attemptOffset === undefined || attemptsCount === undefined) return null
+  return attemptsCount - attemptOffset
+}
+
+export const getAudioStatus = (state: PlayerState): AudioStatus | undefined => {
+  if (state === PlayerState.PLAYING) return AudioStatus.PLAYING
+  if (state === PlayerState.PAUSED) return AudioStatus.PAUSED
+  return undefined
+}
+
+export const is400Error = (error: unknown): boolean => {
+  if (error instanceof Error) {
+    const httpError = error as any
+    return httpError.status === 400 || httpError.response?.status === 400
+  }
+  return false
+}

package/src/features/voice/index.ts

@@ -0,0 +1,11 @@
+export * from './recording/components/VoiceRecord'
+export * from './recording/model/VoiceRecord.model'
+export * from './recording/modal/VoiceRecordDeleteModal'
+export * from './recording/modal/VoiceRecordUndoModal'
+
+export * from './playing/model/VoicePlayer.model'
+export * from './playing/components/VoiceTranscriptionsDropdown'
+
+export * from './playing/components/VoiceIcon'
+export * from './constants'
+export * from './types'

package/src/features/voice/playing/components/VoiceIcon.tsx

@@ -0,0 +1,38 @@
+import { StyleProp, StyleSheet, View, ViewStyle } from 'react-native'
+import React from 'react'
+import { COLORS, MicrophoneIcon } from '@magmamath/react-native-ui'
+import Animated, { FadeIn } from 'react-native-reanimated'
+
+type VoiceIconProps = {
+  style?: StyleProp<ViewStyle>
+}
+
+export const VoiceIcon = ({ style }: VoiceIconProps) => {
+  return (
+    <Animated.View entering={FadeIn.duration(250)} style={[styles.container, style]}>
+      <View style={styles.iconContainer}>
+        <MicrophoneIcon color={COLORS.NEUTRAL_5} />
+      </View>
+    </Animated.View>
+  )
+}
+
+const styles = StyleSheet.create({
+  container: {
+    flex: 1,
+    justifyContent: 'center',
+    alignItems: 'center',
+  },
+  iconContainer: {
+    maxHeight: 124,
+    height: '80%',
+    aspectRatio: 1,
+    alignSelf: 'center',
+    padding: 24,
+    borderWidth: 4,
+    borderColor: COLORS.NEUTRAL_5,
+    borderRadius: 400,
+    justifyContent: 'center',
+    alignItems: 'center',
+  },
+})

package/src/features/voice/playing/components/VoiceTranscriptionsDropdown.tsx

@@ -0,0 +1,256 @@
+import {
+  Typography,
+  ScrollableList,
+  CaretDownIcon,
+  COLORS,
+  IS_WEB,
+} from '@magmamath/react-native-ui'
+import { useUnit } from 'effector-react'
+import React, { useRef, useEffect, useState, useCallback, useMemo } from 'react'
+import { TouchableOpacity, StyleProp, ViewStyle, FlatList, View, StyleSheet } from 'react-native'
+import Animated, { FadeIn, FadeOut } from 'react-native-reanimated'
+import { AnswerAudio, VoiceTranscriptionItem } from '../../types'
+import { useVoiceTranscriptionsDropdown } from '../hooks/useVoiceTranscriptionsDropdown'
+import { VoicePlayerModel, PlayerState } from '../model'
+import { VoiceTranscriptionsDropdownItem } from './VoiceTranscriptionsDropdownItem'
+import { DROPDOWN_MAX_CONTENT_HEIGHT, NO_AUDIO_BE_MESSAGE } from '../../constants'
+import { useText } from '../../../../shared/translation'
+import { PlayButton } from '../../../../shared/components/PlayButton'
+import { getActiveAttempt, getAudioStatus } from '../../helpers'
+
+const AnimatedTouchableOpacity = Animated.createAnimatedComponent(TouchableOpacity)
+
+type VoiceTranscriptionsDropdownProps = {
+  style?: StyleProp<ViewStyle>
+  model: VoicePlayerModel
+  answers: AnswerAudio[]
+  attemptOffset: number
+  onAttemptOffsetChange: (offset: number) => void
+  attemptsCount: number
+  problemId: string
+}
+
+export const VoiceTranscriptionsDropdown = ({
+  style,
+  model,
+  answers,
+  attemptOffset,
+  onAttemptOffsetChange,
+  attemptsCount,
+  problemId,
+}: VoiceTranscriptionsDropdownProps) => {
+  const [attempts, setAttempts] = useState<VoiceTranscriptionItem[]>([])
+
+  const scrollRef = useRef<FlatList | null>(null)
+  const answersRef = useRef(answers)
+  answersRef.current = answers
+
+  const t = useText()
+
+  const [currentAttempt, transcriptsLoaded, playerState, isDropdownExpanded] = useUnit([
+    model.$currentAttempt,
+    model.loader.$transcriptsLoaded,
+    model.$playerState,
+    model.dropdown.$isExpanded,
+  ])
+
+  const activeAttempt = getActiveAttempt(attemptOffset, attemptsCount)
+
+  const answersKey = useMemo(
+    () => answers.map((a) => `${a._id}:${a.audioFileId ?? ''}`).join(','),
+    [answers],
+  )
+
+  const hasActiveTranscript =
+    activeAttempt !== null && answers?.[activeAttempt - 1]?.audioFileId !== undefined
+
+  const playButtonState = {
+    isActive: playerState !== PlayerState.IDLE,
+    isLoading: playerState === PlayerState.LOADING,
+    status: getAudioStatus(playerState),
+  }
+
+  const { listAnimatedStyle, iconAnimatedStyle, onListLayout } = useVoiceTranscriptionsDropdown(
+    model.dropdown,
+  )
+
+  const scrollToAttempt = useCallback(
+    (attemptNumber: number) => {
+      const index = attempts.findIndex((item) => item.attemptNumber === attemptNumber)
+      if (index !== -1) {
+        scrollRef.current?.scrollToIndex({ index, animated: true, viewPosition: 0 })
+      }
+    },
+    [attempts],
+  )
+
+  const handleItemToggle = useCallback(
+    (attempt: number) => {
+      if (onAttemptOffsetChange && attemptsCount !== undefined) {
+        onAttemptOffsetChange(attemptsCount - attempt)
+      }
+    },
+    [onAttemptOffsetChange, attemptsCount],
+  )
+
+  const handlePlayPress = useCallback(() => {
+    if (activeAttempt) {
+      model.togglePlayPause(activeAttempt)
+    }
+  }, [model, activeAttempt])
+
+  const handleScrollToIndexFailed = useCallback((info: { index: number }) => {
+    setTimeout(() => {
+      scrollRef.current?.scrollToIndex({ index: info.index, animated: true, viewPosition: 0 })
+    }, 100)
+  }, [])
+
+  useEffect(() => {
+    model.reinitializePlayer()
+    return () => model.cleanup()
+  }, [model])
+
+  useEffect(() => {
+    const isLastAttemptsWithoutDrowing = !answers[answers.length - 1].drawing
+    if (answers.length > 0 && isLastAttemptsWithoutDrowing) {
+      model.dropdown.setExpanded(true)
+    } else {
+      model.dropdown.setExpanded(false)
+    }
+
+    model.resetPlayer()
+  }, [problemId])
+
+  useEffect(() => {
+    if (answersRef.current.some((answer) => answer.audioFileId)) {
+      model.initializeWithAudios(answersRef.current)
+      model.loader.loadAllTranscripts()
+    }
+  }, [model, answersKey])
+
+  useEffect(() => {
+    model.stopAudio()
+  }, [activeAttempt])
+
+  useEffect(() => {
+    const allAttempts = model.getAllAttempts()
+    setAttempts(allAttempts)
+  }, [answersKey, transcriptsLoaded, currentAttempt, model])
+
+  useEffect(() => {
+    if (isDropdownExpanded && activeAttempt !== null) {
+      setTimeout(() => {
+        scrollToAttempt(activeAttempt)
+      }, 500)
+    }
+  }, [isDropdownExpanded, answersKey])
+
+  const renderItem = useCallback(
+    ({ item }: { item: VoiceTranscriptionItem }) => {
+      const isLoading = model.loader.isTranscriptLoading(item.attemptNumber)
+      const transcript = model.loader.getTranscript(item.attemptNumber)
+      const isActive = activeAttempt === item.attemptNumber
+
+      return (
+        <VoiceTranscriptionsDropdownItem
+          isLoading={isActive && isLoading}
+          attempt={item.attemptNumber}
+          content={
+            transcript && transcript !== NO_AUDIO_BE_MESSAGE
+              ? transcript
+              : t('voice.transcriptNotAvailable')
+          }
+          isActive={isActive}
+          onToggle={handleItemToggle}
+          scrollToAttempt={scrollToAttempt}
+        />
+      )
+    },
+    [model, activeAttempt, t, handleItemToggle, scrollToAttempt],
+  )
+
+  return (
+    <Animated.View style={[styles.container, style]}>
+      <View style={styles.header}>
+        <Typography style={styles.title} variant="h7">
+          {t('voice.transcriptions')}
+        </Typography>
+        {hasActiveTranscript && (
+          <Animated.View entering={FadeIn.duration(120)} exiting={FadeOut.duration(120)}>
+            <PlayButton
+              state={playButtonState}
+              onPress={handlePlayPress}
+              activityIndicatorColor={COLORS.NEUTRAL_1}
+            />
+          </Animated.View>
+        )}
+      </View>
+
+      <Animated.View style={[listAnimatedStyle, { overflow: 'hidden' }]}>
+        <View onLayout={onListLayout} style={styles.listContainer}>
+          <ScrollableList
+            bounces={false}
+            ref={scrollRef}
+            hideShadow
+            data={attempts}
+            keyExtractor={(item) => item._id}
+            onScrollToIndexFailed={handleScrollToIndexFailed}
+            renderItem={renderItem}
+            style={IS_WEB && styles.list}
+            contentContainerStyle={styles.listContent}
+            //for some resons on web FlatList doesn't render last items in huge list, so we need to render them all at once
+            initialNumToRender={IS_WEB ? 20 : undefined}
+          />
+        </View>
+      </Animated.View>
+
+      <AnimatedTouchableOpacity
+        style={[styles.expandButton, iconAnimatedStyle]}
+        onPress={() => model.dropdown.toggleExpand()}
+      >
+        <CaretDownIcon size={22} />
+      </AnimatedTouchableOpacity>
+    </Animated.View>
+  )
+}
+
+const styles = StyleSheet.create({
+  container: {
+    position: 'absolute',
+    top: 56,
+    left: 8,
+    width: 222,
+    maxHeight: 405,
+    minHeight: 51,
+    padding: 4,
+    boxShadow: '0px 1px 3px 0px rgba(51, 51, 51, 0.25)',
+    borderRadius: 12,
+    backgroundColor: COLORS.NEUTRAL_1,
+    overflow: 'hidden',
+  },
+  header: {
+    flexDirection: 'row',
+    justifyContent: 'space-between',
+    alignItems: 'center',
+    padding: 4,
+    marginBottom: 4,
+    height: 32,
+  },
+  title: {
+    color: COLORS.NEUTRAL_9,
+    marginLeft: 4,
+  },
+  list: { flex: 1 },
+  expandButton: {
+    justifyContent: 'center',
+    alignItems: 'center',
+  },
+  listContainer: {
+    position: 'absolute',
+    maxHeight: DROPDOWN_MAX_CONTENT_HEIGHT,
+    width: '100%',
+  },
+  listContent: {
+    paddingHorizontal: 4,
+  },
+})