nastechai-desktop 18.1.0

package/src/app/chat/composer/hooks/use-voice-conversation.ts

@@ -0,0 +1,390 @@
+import { useCallback, useEffect, useRef, useState } from 'react'
+
+import { useI18n } from '@/i18n'
+import { playSpeechText, stopVoicePlayback } from '@/lib/voice-playback'
+import { notify, notifyError } from '@/store/notifications'
+
+import { useMicRecorder } from './use-mic-recorder'
+
+export type ConversationStatus = 'idle' | 'listening' | 'transcribing' | 'thinking' | 'speaking'
+
+interface PendingVoiceResponse {
+  id: string
+  pending: boolean
+  text: string
+}
+
+interface VoiceConversationOptions {
+  busy: boolean
+  enabled: boolean
+  onFatalError?: () => void
+  onSubmit: (text: string) => Promise<void> | void
+  onTranscribeAudio?: (audio: Blob) => Promise<string>
+  pendingResponse: () => PendingVoiceResponse | null
+  consumePendingResponse: () => void
+}
+
+export function useVoiceConversation({
+  busy,
+  enabled,
+  onFatalError,
+  onSubmit,
+  onTranscribeAudio,
+  pendingResponse,
+  consumePendingResponse
+}: VoiceConversationOptions) {
+  const { t } = useI18n()
+  const voiceCopy = t.notifications.voice
+  const { handle, level } = useMicRecorder(voiceCopy)
+  const [status, setStatus] = useState<ConversationStatus>('idle')
+  const [muted, setMuted] = useState(false)
+  const turnTimeoutRef = useRef<number | null>(null)
+  const pendingStartRef = useRef(false)
+  const turnClosingRef = useRef(false)
+  const awaitingSpokenResponseRef = useRef(false)
+  const responseIdRef = useRef<string | null>(null)
+  const spokenSourceLengthRef = useRef(0)
+  const speechBufferRef = useRef('')
+  const enabledRef = useRef(enabled)
+  const mutedRef = useRef(muted)
+  const busyRef = useRef(busy)
+  const statusRef = useRef<ConversationStatus>('idle')
+  const wasEnabledRef = useRef(enabled)
+
+  useEffect(() => {
+    enabledRef.current = enabled
+  }, [enabled])
+
+  useEffect(() => {
+    mutedRef.current = muted
+  }, [muted])
+
+  useEffect(() => {
+    busyRef.current = busy
+  }, [busy])
+
+  useEffect(() => {
+    statusRef.current = status
+  }, [status])
+
+  const clearTurnTimeout = () => {
+    if (turnTimeoutRef.current) {
+      window.clearTimeout(turnTimeoutRef.current)
+      turnTimeoutRef.current = null
+    }
+  }
+
+  const resetSpeechBuffer = () => {
+    responseIdRef.current = null
+    spokenSourceLengthRef.current = 0
+    speechBufferRef.current = ''
+  }
+
+  const appendSpeechText = (text: string) => {
+    if (!text) {
+      return
+    }
+
+    speechBufferRef.current = `${speechBufferRef.current}${text}`
+  }
+
+  const takeSpeechChunk = (force = false): string | null => {
+    const buffer = speechBufferRef.current.replace(/\s+/g, ' ').trim()
+
+    if (!buffer) {
+      speechBufferRef.current = ''
+
+      return null
+    }
+
+    const sentence = buffer.match(/^(.+?[.!?。！？])(?:\s+|$)/)
+
+    if (sentence?.[1] && (sentence[1].length >= 8 || force)) {
+      const chunk = sentence[1].trim()
+      speechBufferRef.current = buffer.slice(sentence[1].length).trim()
+
+      return chunk
+    }
+
+    if (!force && buffer.length > 220) {
+      const softBoundary = Math.max(
+        buffer.lastIndexOf(', ', 180),
+        buffer.lastIndexOf('; ', 180),
+        buffer.lastIndexOf(': ', 180)
+      )
+
+      if (softBoundary > 80) {
+        const chunk = buffer.slice(0, softBoundary + 1).trim()
+        speechBufferRef.current = buffer.slice(softBoundary + 1).trim()
+
+        return chunk
+      }
+    }
+
+    if (!force) {
+      return null
+    }
+
+    speechBufferRef.current = ''
+
+    return buffer
+  }
+
+  const handleTurn = useCallback(
+    async (forceTranscribe = false) => {
+      if (turnClosingRef.current) {
+        return
+      }
+
+      turnClosingRef.current = true
+      clearTurnTimeout()
+      setStatus('transcribing')
+
+      try {
+        const result = await handle.stop()
+
+        if (!result || (!result.heardSpeech && !forceTranscribe) || !onTranscribeAudio) {
+          if (enabledRef.current && !mutedRef.current && !busyRef.current && statusRef.current !== 'speaking') {
+            pendingStartRef.current = true
+          }
+
+          setStatus('idle')
+
+          return
+        }
+
+        try {
+          const transcript = (await onTranscribeAudio(result.audio)).trim()
+
+          if (!transcript) {
+            if (enabledRef.current) {
+              pendingStartRef.current = true
+            }
+
+            setStatus('idle')
+
+            return
+          }
+
+          awaitingSpokenResponseRef.current = true
+          resetSpeechBuffer()
+          await onSubmit(transcript)
+          setStatus('thinking')
+        } catch (error) {
+          notifyError(error, voiceCopy.transcriptionFailed)
+
+          if (enabledRef.current && !mutedRef.current && !busyRef.current) {
+            pendingStartRef.current = true
+          }
+
+          setStatus('idle')
+        }
+      } finally {
+        turnClosingRef.current = false
+      }
+    },
+    [handle, onSubmit, onTranscribeAudio, voiceCopy.transcriptionFailed]
+  )
+
+  const startListening = useCallback(async () => {
+    pendingStartRef.current = false
+
+    if (!enabledRef.current || mutedRef.current || busyRef.current) {
+      return
+    }
+
+    if (statusRef.current !== 'idle') {
+      return
+    }
+
+    try {
+      // VAD tuning mirrors `tools.voice_mode` defaults so the browser loop matches the CLI.
+      await handle.start({
+        silenceLevel: 0.075,
+        silenceMs: 1_250,
+        idleSilenceMs: 12_000,
+        onError: error => {
+          notifyError(error, voiceCopy.microphoneFailed)
+          pendingStartRef.current = false
+          onFatalError?.()
+        },
+        onSilence: () => void handleTurn()
+      })
+      setStatus('listening')
+      turnTimeoutRef.current = window.setTimeout(() => void handleTurn(), 60_000)
+    } catch (error) {
+      notifyError(error, voiceCopy.couldNotStartSession)
+      pendingStartRef.current = false
+      setStatus('idle')
+      onFatalError?.()
+    }
+  }, [handle, handleTurn, onFatalError, voiceCopy.couldNotStartSession, voiceCopy.microphoneFailed])
+
+  const speak = useCallback(async (text: string) => {
+    setStatus('speaking')
+
+    try {
+      await playSpeechText(text, { source: 'voice-conversation' })
+    } catch (error) {
+      notifyError(error, voiceCopy.playbackFailed)
+    } finally {
+      if (enabledRef.current) {
+        pendingStartRef.current = true
+        setStatus('idle')
+      } else {
+        setStatus('idle')
+      }
+    }
+  }, [voiceCopy.playbackFailed])
+
+  const start = useCallback(async () => {
+    if (!onTranscribeAudio) {
+      notify({
+        kind: 'warning',
+        title: voiceCopy.unavailable,
+        message: voiceCopy.configureSpeechToText
+      })
+      onFatalError?.()
+
+      return
+    }
+
+    setMuted(false)
+    awaitingSpokenResponseRef.current = false
+    resetSpeechBuffer()
+    consumePendingResponse()
+    pendingStartRef.current = true
+    await startListening()
+  }, [consumePendingResponse, onFatalError, onTranscribeAudio, startListening, voiceCopy.configureSpeechToText, voiceCopy.unavailable])
+
+  const end = useCallback(async () => {
+    pendingStartRef.current = false
+    clearTurnTimeout()
+    stopVoicePlayback()
+    handle.cancel()
+    turnClosingRef.current = false
+    awaitingSpokenResponseRef.current = false
+    resetSpeechBuffer()
+    consumePendingResponse()
+    setMuted(false)
+    setStatus('idle')
+  }, [consumePendingResponse, handle])
+
+  const stopTurn = useCallback(() => {
+    if (statusRef.current === 'listening') {
+      void handleTurn(true)
+    }
+  }, [handleTurn])
+
+  const toggleMute = useCallback(() => {
+    setMuted(value => {
+      const next = !value
+
+      if (next) {
+        clearTurnTimeout()
+        handle.cancel()
+        setStatus('idle')
+      } else if (enabledRef.current && !busyRef.current && statusRef.current === 'idle') {
+        pendingStartRef.current = true
+      }
+
+      return next
+    })
+  }, [handle])
+
+  useEffect(() => {
+    if (!enabled) {
+      return
+    }
+
+    const onKeyDown = (event: KeyboardEvent) => {
+      if (event.code !== 'Space' || event.repeat || event.metaKey || event.ctrlKey || event.altKey) {
+        return
+      }
+
+      if (statusRef.current !== 'listening') {
+        return
+      }
+
+      event.preventDefault()
+      stopTurn()
+    }
+
+    window.addEventListener('keydown', onKeyDown, { capture: true })
+
+    return () => window.removeEventListener('keydown', onKeyDown, { capture: true })
+  }, [enabled, stopTurn])
+
+  // Drive the loop: after a voice-submitted turn, speak stable chunks as the
+  // assistant stream grows. Otherwise start listening when idle between turns.
+  useEffect(() => {
+    if (!enabled || muted) {
+      return
+    }
+
+    if (awaitingSpokenResponseRef.current && status !== 'speaking') {
+      const response = pendingResponse()
+
+      if (response) {
+        if (response.id !== responseIdRef.current) {
+          resetSpeechBuffer()
+          responseIdRef.current = response.id
+        }
+
+        if (response.text.length > spokenSourceLengthRef.current) {
+          appendSpeechText(response.text.slice(spokenSourceLengthRef.current))
+          spokenSourceLengthRef.current = response.text.length
+        }
+
+        const chunk = takeSpeechChunk(!response.pending && !busy)
+
+        if (chunk) {
+          void speak(chunk)
+
+          return
+        }
+
+        if (!response.pending && !busy) {
+          awaitingSpokenResponseRef.current = false
+          consumePendingResponse()
+          resetSpeechBuffer()
+          pendingStartRef.current = true
+          setStatus('idle')
+
+          return
+        }
+      }
+
+      if (!busy && status === 'thinking') {
+        awaitingSpokenResponseRef.current = false
+        resetSpeechBuffer()
+        pendingStartRef.current = true
+        setStatus('idle')
+
+        return
+      }
+    }
+
+    if (busy || status !== 'idle') {
+      return
+    }
+
+    if (pendingStartRef.current) {
+      void startListening()
+    }
+  }, [busy, consumePendingResponse, enabled, muted, pendingResponse, speak, startListening, status])
+
+  useEffect(() => {
+    if (enabled && !wasEnabledRef.current) {
+      void start()
+    }
+
+    if (!enabled && wasEnabledRef.current) {
+      void end()
+    }
+
+    wasEnabledRef.current = enabled
+  }, [enabled, end, start])
+
+  return { end, level, muted, start, status, stopTurn, toggleMute }
+}

package/src/app/chat/composer/hooks/use-voice-recorder.ts

@@ -0,0 +1,116 @@
+import { useEffect, useRef, useState } from 'react'
+
+import { useI18n } from '@/i18n'
+import { notify, notifyError } from '@/store/notifications'
+
+import type { VoiceActivityState, VoiceStatus } from '../types'
+
+import { useMicRecorder } from './use-mic-recorder'
+
+interface VoiceRecorderOptions {
+  maxRecordingSeconds: number
+  onTranscribeAudio?: (audio: Blob) => Promise<string>
+  focusInput: () => void
+  onTranscript: (text: string) => void
+}
+
+export function useVoiceRecorder({
+  maxRecordingSeconds,
+  onTranscribeAudio,
+  focusInput,
+  onTranscript
+}: VoiceRecorderOptions) {
+  const { t } = useI18n()
+  const voiceCopy = t.notifications.voice
+  const { handle, level, recording } = useMicRecorder(voiceCopy)
+  const [voiceStatus, setVoiceStatus] = useState<VoiceStatus>('idle')
+  const [elapsedSeconds, setElapsedSeconds] = useState(0)
+  const startedAtRef = useRef(0)
+  const intervalRef = useRef<number | null>(null)
+  const timeoutRef = useRef<number | null>(null)
+
+  const clearTimers = () => {
+    if (intervalRef.current) {
+      window.clearInterval(intervalRef.current)
+      intervalRef.current = null
+    }
+
+    if (timeoutRef.current) {
+      window.clearTimeout(timeoutRef.current)
+      timeoutRef.current = null
+    }
+  }
+
+  useEffect(() => () => clearTimers(), [])
+
+  const stop = async () => {
+    clearTimers()
+    const result = await handle.stop()
+
+    if (!result) {
+      setVoiceStatus('idle')
+
+      return
+    }
+
+    if (!onTranscribeAudio) {
+      setVoiceStatus('idle')
+
+      return
+    }
+
+    setVoiceStatus('transcribing')
+
+    try {
+      const transcript = (await onTranscribeAudio(result.audio)).trim()
+
+      if (!transcript) {
+        notify({ kind: 'warning', title: voiceCopy.noSpeechDetected, message: voiceCopy.tryRecordingAgain })
+      } else {
+        onTranscript(transcript)
+      }
+    } catch (error) {
+      notifyError(error, voiceCopy.transcriptionFailed)
+    } finally {
+      setVoiceStatus('idle')
+      focusInput()
+    }
+  }
+
+  const start = async () => {
+    if (!onTranscribeAudio) {
+      notify({ kind: 'warning', title: voiceCopy.unavailable, message: voiceCopy.transcriptionUnavailable })
+
+      return
+    }
+
+    try {
+      await handle.start({ onError: error => notifyError(error, voiceCopy.recordingFailed) })
+      startedAtRef.current = Date.now()
+      setElapsedSeconds(0)
+      setVoiceStatus('recording')
+      intervalRef.current = window.setInterval(() => setElapsedSeconds((Date.now() - startedAtRef.current) / 1000), 250)
+      const cap = Math.max(1, Math.min(Math.trunc(maxRecordingSeconds), 600))
+      timeoutRef.current = window.setTimeout(() => void stop(), cap * 1000)
+    } catch (error) {
+      setVoiceStatus('idle')
+      notifyError(error, voiceCopy.recordingFailed)
+    }
+  }
+
+  const dictate = () => {
+    if (recording) {
+      void stop()
+    } else if (voiceStatus === 'idle') {
+      void start()
+    }
+  }
+
+  const voiceActivityState: VoiceActivityState = {
+    elapsedSeconds,
+    level,
+    status: voiceStatus
+  }
+
+  return { dictate, voiceActivityState, voiceStatus }
+}

package/src/app/chat/composer/ime-composition-dom-repro.test.tsx

@@ -0,0 +1,108 @@
+import { act, cleanup, fireEvent, render } from '@testing-library/react'
+import { useRef, useState } from 'react'
+import { afterEach, describe, expect, it } from 'vitest'
+
+// No global setupFiles registers auto-cleanup, so unmount between tests —
+// otherwise a second render() leaks the first editor and getByTestId('editor')
+// matches multiple nodes.
+afterEach(cleanup)
+
+// Faithful mirror of index.tsx's composer text wiring for IME input, driven
+// through REAL DOM composition + input events on a contentEditable.
+//
+// Regression repro for #39614: typing committed multi-character IME text (e.g.
+// Chinese "你好") used to leave the send button hidden. The input events fired
+// during composition carry uncommitted preedit text and are intentionally
+// skipped; Chromium then does NOT reliably emit a trailing input event after
+// compositionend on Windows IMEs, so the finalized text never reached composer
+// state and `hasPayload` stayed false until an unrelated edit forced a sync.
+// The fix flushes the live DOM text in onCompositionEnd.
+function Harness({ onPayload }: { onPayload: (hasPayload: boolean) => void }) {
+  const editorRef = useRef<HTMLDivElement>(null)
+  const composingRef = useRef(false)
+  const draftRef = useRef('')
+  const [draft, setDraft] = useState('')
+
+  const flushEditorToDraft = (editor: HTMLDivElement) => {
+    const next = editor.textContent ?? ''
+
+    if (next !== draftRef.current) {
+      draftRef.current = next
+      setDraft(next)
+    }
+  }
+
+  onPayload(draft.trim().length > 0)
+
+  return (
+    <div
+      contentEditable
+      data-testid="editor"
+      onCompositionEnd={event => {
+        composingRef.current = false
+        flushEditorToDraft(event.currentTarget)
+      }}
+      onCompositionStart={() => {
+        composingRef.current = true
+      }}
+      onInput={event => {
+        if (composingRef.current) {
+          return
+        }
+
+        flushEditorToDraft(event.currentTarget)
+      }}
+      ref={editorRef}
+      suppressContentEditableWarning
+    />
+  )
+}
+
+describe('composer IME composition — send button visibility (#39614)', () => {
+  it('shows the send button after committing CJK text without a trailing edit', async () => {
+    let hasPayload = false
+    const { getByTestId } = render(<Harness onPayload={p => (hasPayload = p)} />)
+    const editor = getByTestId('editor')
+
+    // Compose "你好" the way a Windows Chinese IME does: compositionstart, then
+    // input events carrying uncommitted preedit text, then compositionend with
+    // the committed text already in the DOM — and crucially NO input event
+    // afterwards.
+    await act(async () => {
+      fireEvent.compositionStart(editor)
+      editor.textContent = '你'
+      fireEvent.input(editor)
+      editor.textContent = '你好'
+      fireEvent.input(editor)
+      fireEvent.compositionEnd(editor)
+    })
+
+    // Before the fix this was false (button hidden) until a further edit.
+    expect(hasPayload).toBe(true)
+    expect(editor.textContent).toBe('你好')
+  })
+
+  it('also covers Japanese/Korean and any IME-composed script', async () => {
+    let hasPayload = false
+    const { getByTestId } = render(<Harness onPayload={p => (hasPayload = p)} />)
+    const editor = getByTestId('editor')
+
+    for (const committed of ['こんにちは', '안녕하세요']) {
+      await act(async () => {
+        fireEvent.compositionStart(editor)
+        editor.textContent = committed
+        fireEvent.input(editor)
+        fireEvent.compositionEnd(editor)
+      })
+
+      expect(hasPayload).toBe(true)
+
+      // Clear for the next script.
+      await act(async () => {
+        editor.textContent = ''
+        fireEvent.input(editor)
+      })
+      expect(hasPayload).toBe(false)
+    }
+  })
+})