@rimori/client 1.4.0 → 1.4.3

package/src/components/ai/EmbeddedAssistent/AudioInputField.tsx

@@ -1,64 +1,73 @@
 import React, { useState } from 'react';
 import { VoiceRecorder } from './VoiceRecoder';
-import { BiSolidRightArrow } from "react-icons/bi";
-import { HiMiniSpeakerXMark, HiMiniSpeakerWave } from "react-icons/hi2";
+import { BiSolidRightArrow } from 'react-icons/bi';
+import { HiMiniSpeakerXMark, HiMiniSpeakerWave } from 'react-icons/hi2';
 
 interface AudioInputFieldProps {
-    onSubmit: (text: string) => void;
-    onAudioControl?: (voice: boolean) => void;
-    blockSubmission?: boolean;
+  onSubmit: (text: string) => void;
+  onAudioControl?: (voice: boolean) => void;
+  blockSubmission?: boolean;
 }
 
 export function AudioInputField({ onSubmit, onAudioControl, blockSubmission = false }: AudioInputFieldProps) {
-    const [text, setText] = useState('');
-    const [audioEnabled, setAudioEnabled] = useState(true);
+  const [text, setText] = useState('');
+  const [audioEnabled, setAudioEnabled] = useState(true);
 
-    const handleSubmit = (manualText?: string) => {
-        if (blockSubmission) return;
-        const sendableText = manualText || text;
-        if (sendableText.trim()) {
-            onSubmit(sendableText);
-            setTimeout(() => {
-                setText('');
-            }, 100);
-        }
-    };
+  const handleSubmit = (manualText?: string) => {
+    if (blockSubmission) return;
+    const sendableText = manualText || text;
+    if (sendableText.trim()) {
+      onSubmit(sendableText);
+      setTimeout(() => {
+        setText('');
+      }, 100);
+    }
+  };
 
-    const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
-        if (blockSubmission) return;
-        if (e.key === 'Enter' && e.ctrlKey) {
-            setText(text + '\n');
-        } else if (e.key === 'Enter') {
-            handleSubmit();
-        }
-    };
+  const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
+    if (blockSubmission) return;
+    if (e.key === 'Enter' && e.ctrlKey) {
+      setText(text + '\n');
+    } else if (e.key === 'Enter') {
+      handleSubmit();
+    }
+  };
 
-    return (
-        <div className="flex items-center bg-gray-600 pt-2 pb-2 p-2">
-            {onAudioControl && <button
-                onClick={() => {
-                    onAudioControl(!audioEnabled);
-                    setAudioEnabled(!audioEnabled);
-                }}
-                className="cursor-default">
-                {audioEnabled ? <HiMiniSpeakerWave className='w-9 h-9 cursor-pointer' /> : <HiMiniSpeakerXMark className='w-9 h-9 cursor-pointer' />}
-            </button>}
-            <VoiceRecorder onRecordingStatusChange={() => {}} onVoiceRecorded={(m: string) => {
-                console.log('onVoiceRecorded', m);
-                handleSubmit(m);
-            }}
-            />
-            <textarea
-                value={text}
-                onChange={(e) => setText(e.target.value)}
-                onKeyDown={handleKeyDown}
-                className="flex-1 border-none rounded-lg p-2 text-gray-800 focus::outline-none"
-                placeholder='Type a message...'
-                disabled={blockSubmission}
-            />
-            <button onClick={() => handleSubmit()} className="cursor-default" disabled={blockSubmission}>
-                <BiSolidRightArrow className='w-9 h-10 cursor-pointer' />
-            </button>
-        </div>
-    );
-};
+  return (
+    <div className="flex items-center bg-gray-600 pt-2 pb-2 p-2">
+      {onAudioControl && (
+        <button
+          onClick={() => {
+            onAudioControl(!audioEnabled);
+            setAudioEnabled(!audioEnabled);
+          }}
+          className="cursor-default"
+        >
+          {audioEnabled ? (
+            <HiMiniSpeakerWave className="w-9 h-9 cursor-pointer" />
+          ) : (
+            <HiMiniSpeakerXMark className="w-9 h-9 cursor-pointer" />
+          )}
+        </button>
+      )}
+      <VoiceRecorder
+        onRecordingStatusChange={() => {}}
+        onVoiceRecorded={(m: string) => {
+          console.log('onVoiceRecorded', m);
+          handleSubmit(m);
+        }}
+      />
+      <textarea
+        value={text}
+        onChange={(e) => setText(e.target.value)}
+        onKeyDown={handleKeyDown}
+        className="flex-1 border-none rounded-lg p-2 text-gray-800 focus::outline-none"
+        placeholder="Type a message..."
+        disabled={blockSubmission}
+      />
+      <button onClick={() => handleSubmit()} className="cursor-default" disabled={blockSubmission}>
+        <BiSolidRightArrow className="w-9 h-10 cursor-pointer" />
+      </button>
+    </div>
+  );
+}

package/src/components/ai/EmbeddedAssistent/CircleAudioAvatar.tsx

@@ -8,7 +8,12 @@ interface CircleAudioAvatarProps {
   isDarkTheme?: boolean;
 }
 
-export function CircleAudioAvatar({ imageUrl, className, isDarkTheme = false, width = "150px" }: CircleAudioAvatarProps) {
+export function CircleAudioAvatar({
+  imageUrl,
+  className,
+  isDarkTheme = false,
+  width = '150px',
+}: CircleAudioAvatarProps) {
   const canvasRef = useRef<HTMLCanvasElement>(null);
   const currentLoudnessRef = useRef(0);
   const targetLoudnessRef = useRef(0);
@@ -31,7 +36,7 @@ export function CircleAudioAvatar({ imageUrl, className, isDarkTheme = false, wi
             if (currentLoudnessRef.current > targetLoudnessRef.current) {
               currentLoudnessRef.current = Math.max(
                 targetLoudnessRef.current,
-                currentLoudnessRef.current - decayRate * currentLoudnessRef.current
+                currentLoudnessRef.current - decayRate * currentLoudnessRef.current,
               );
             } else {
               currentLoudnessRef.current = targetLoudnessRef.current;
@@ -63,7 +68,12 @@ export function CircleAudioAvatar({ imageUrl, className, isDarkTheme = false, wi
     }
   }, [imageUrl]);
 
-  const draw = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, image: HTMLImageElement, loudness: number) => {
+  const draw = (
+    ctx: CanvasRenderingContext2D,
+    canvas: HTMLCanvasElement,
+    image: HTMLImageElement,
+    loudness: number,
+  ) => {
     if (canvas && ctx) {
       ctx.clearRect(0, 0, canvas.width, canvas.height);
 
@@ -94,5 +104,4 @@ export function CircleAudioAvatar({ imageUrl, className, isDarkTheme = false, wi
   };
 
   return <canvas ref={canvasRef} className={className} width={500} height={500} style={{ width }} />;
-};
-
+}

package/src/components/ai/EmbeddedAssistent/TTS/MessageSender.ts

@@ -3,93 +3,94 @@ import { ChunkedAudioPlayer } from './Player';
 type VoiceBackend = (text: string, voice?: string, speed?: number) => Promise<Blob>;
 
 export class MessageSender {
-    private player = new ChunkedAudioPlayer();
-    private fetchedSentences = new Set<string>();
-    private lastLoading = false;
-    private voice: string;
-    private model: string;
-    private voiceBackend: VoiceBackend;
+  private player = new ChunkedAudioPlayer();
+  private fetchedSentences = new Set<string>();
+  private lastLoading = false;
+  private voice: string;
+  private voiceBackend: VoiceBackend;
 
-    constructor(voiceBackend: VoiceBackend, voice: string = 'alloy', model = 'openai') {
-        this.voiceBackend = voiceBackend;
-        this.voice = voice;
-        this.model = model;
+  constructor(voiceBackend: VoiceBackend, voice: string) {
+    if (voice?.split('_').length !== 2) {
+      throw new Error("Invalid voice id format '" + voice + "'. Voice id needs to look like <provider>_<voice_id>");
     }
+    this.voiceBackend = voiceBackend;
+    this.voice = voice;
+  }
 
-    private getCompletedSentences(currentText: string, isLoading: boolean): string[] {
-        // Split the text based on the following characters: .,?!
-        // Only split on : when followed by a space
-        const pattern = /(.+?[,.?!]|.+?:\s+|.+?\n+)/g;
-        const result: string[] = [];
-        let match;
-        while ((match = pattern.exec(currentText)) !== null) {
-            const sentence = match[0].trim();
-            if (sentence.length > 0) {
-                result.push(sentence);
-            }
-        }
-        if (!isLoading) {
-            const lastFullSentence = result[result.length - 1];
-            const leftoverIndex = currentText.lastIndexOf(lastFullSentence) + lastFullSentence.length;
-            if (leftoverIndex < currentText.length) {
-                result.push(currentText.slice(leftoverIndex).trim());
-            }
-        }
-        return result;
+  private getCompletedSentences(currentText: string, isLoading: boolean): string[] {
+    // Split the text based on the following characters: .,?!
+    // Only split on : when followed by a space
+    const pattern = /(.+?[,.?!]|.+?:\s+|.+?\n+)/g;
+    const result: string[] = [];
+    let match;
+    while ((match = pattern.exec(currentText)) !== null) {
+      const sentence = match[0].trim();
+      if (sentence.length > 0) {
+        result.push(sentence);
+      }
     }
+    if (!isLoading) {
+      const lastFullSentence = result[result.length - 1];
+      const leftoverIndex = currentText.lastIndexOf(lastFullSentence) + lastFullSentence.length;
+      if (leftoverIndex < currentText.length) {
+        result.push(currentText.slice(leftoverIndex).trim());
+      }
+    }
+    return result;
+  }
 
-    public async handleNewText(currentText: string | undefined, isLoading: boolean) {
-        if (!this.lastLoading && isLoading) {
-            this.reset();
-        }
-        this.lastLoading = isLoading;
+  public async handleNewText(currentText: string | undefined, isLoading: boolean) {
+    if (!this.lastLoading && isLoading) {
+      this.reset();
+    }
+    this.lastLoading = isLoading;
 
-        if (!currentText) {
-            return;
-        }
+    if (!currentText) {
+      return;
+    }
 
-        const sentences = this.getCompletedSentences(currentText, isLoading);
+    const sentences = this.getCompletedSentences(currentText, isLoading);
 
-        for (let i = 0; i < sentences.length; i++) {
-            const sentence = sentences[i];
-            if (!this.fetchedSentences.has(sentence)) {
-                this.fetchedSentences.add(sentence);
-                const audioData = await this.generateSpeech(sentence);
-                await this.player.addChunk(audioData, i);
-            }
-        }
+    for (let i = 0; i < sentences.length; i++) {
+      const sentence = sentences[i];
+      if (!this.fetchedSentences.has(sentence)) {
+        this.fetchedSentences.add(sentence);
+        const audioData = await this.generateSpeech(sentence);
+        await this.player.addChunk(audioData, i);
+      }
     }
+  }
 
-    private async generateSpeech(sentence: string): Promise<ArrayBuffer> {
-        const blob = await this.voiceBackend(sentence, this.voice, 1.0);
-        return await blob.arrayBuffer();
-    }
+  private async generateSpeech(sentence: string): Promise<ArrayBuffer> {
+    const blob = await this.voiceBackend(sentence, this.voice, 1.0);
+    return await blob.arrayBuffer();
+  }
 
-    public play() {
-        this.player.playAgain();
-    }
+  public play() {
+    this.player.playAgain();
+  }
 
-    public stop() {
-        this.player.stopPlayback();
-    }
+  public stop() {
+    this.player.stopPlayback();
+  }
 
-    private reset() {
-        this.stop();
-        this.fetchedSentences.clear();
-        this.player.reset();
-    }
+  private reset() {
+    this.stop();
+    this.fetchedSentences.clear();
+    this.player.reset();
+  }
 
-    public setVolume(volume: number) {
-        this.player.setVolume(volume);
-    }
+  public setVolume(volume: number) {
+    this.player.setVolume(volume);
+  }
 
-    public setOnLoudnessChange(callback: (value: number) => void) {
-        this.player.setOnLoudnessChange((loudness) => {
-            callback(loudness);
-        });
-    }
+  public setOnLoudnessChange(callback: (value: number) => void) {
+    this.player.setOnLoudnessChange((loudness) => {
+      callback(loudness);
+    });
+  }
 
-    public setOnEndOfSpeech(callback: () => void) {
-        this.player.setOnEndOfSpeech(callback);
-    }
-}
+  public setOnEndOfSpeech(callback: () => void) {
+    this.player.setOnEndOfSpeech(callback);
+  }
+}

package/src/components/ai/EmbeddedAssistent/TTS/Player.ts

@@ -1,5 +1,4 @@
 export class ChunkedAudioPlayer {
-
   private audioContext!: AudioContext;
   private chunkQueue: ArrayBuffer[] = [];
   private isPlaying = false;
@@ -9,10 +8,10 @@ export class ChunkedAudioPlayer {
   private isMonitoring = false;
   private handle = 0;
   private volume = 1.0;
-  private loudnessCallback: (value: number) => void = () => { };
+  private loudnessCallback: (value: number) => void = () => {};
   private currentIndex = 0;
   private startedPlaying = false;
-  private onEndOfSpeech: () => void = () => { };
+  private onEndOfSpeech: () => void = () => {};
 
   constructor() {
     this.init();
@@ -195,4 +194,4 @@ export class ChunkedAudioPlayer {
   public setOnEndOfSpeech(callback: () => void) {
     this.onEndOfSpeech = callback;
   }
-}
+}

package/src/components/ai/EmbeddedAssistent/VoiceRecoder.tsx

@@ -13,102 +13,117 @@ interface Props {
   onVoiceRecorded: (message: string) => void;
 }
 
-export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className, disabled, loading, onRecordingStatusChange, enablePushToTalk = false }: Props, ref) => {
-  const [isRecording, setIsRecording] = useState(false);
-  const [internalIsProcessing, setInternalIsProcessing] = useState(false);
-  const audioControllerRef = useRef<AudioController | null>(null);
-  const { ai, plugin } = useRimori();
-
-  // Ref for latest onVoiceRecorded callback
-  const onVoiceRecordedRef = useRef(onVoiceRecorded);
-  useEffect(() => {
-    onVoiceRecordedRef.current = onVoiceRecorded;
-  }, [onVoiceRecorded]);
-
-  const startRecording = async () => {
-    try {
-      if (!audioControllerRef.current) {
-        audioControllerRef.current = new AudioController(plugin.pluginId);
-      }
-
-      await audioControllerRef.current.startRecording();
-      setIsRecording(true);
-      onRecordingStatusChange(true);
-    } catch (error) {
-      console.error('Failed to start recording:', error);
-      // Handle permission denied or other errors
-    }
-  };
-
-
-
-  const stopRecording = async () => {
-    try {
-      if (audioControllerRef.current && isRecording) {
-        const audioResult = await audioControllerRef.current.stopRecording();
-        // console.log("audioResult: ", audioResult);
-
-        setInternalIsProcessing(true);
-
-        // Play the recorded audio from the Blob
-        // const blobUrl = URL.createObjectURL(audioResult.recording);
-        // const audioRef = new Audio(blobUrl);
-        // audioRef.onended = () => URL.revokeObjectURL(blobUrl);
-        // audioRef.play().catch((e) => console.error('Playback error:', e));
-
-        // console.log("audioBlob: ", audioResult.recording);
-        const text = await ai.getTextFromVoice(audioResult.recording);
-        // console.log("stt result", text);
-        // throw new Error("test");
-        setInternalIsProcessing(false);
-        onVoiceRecordedRef.current(text);
-      }
-    } catch (error) {
-      console.error('Failed to stop recording:', error);
-    } finally {
-      setIsRecording(false);
-      onRecordingStatusChange(false);
-    }
-  };
-
-  useImperativeHandle(ref, () => ({
-    startRecording,
-    stopRecording,
-  }));
-
-  // push to talk feature
-  const spacePressedRef = useRef(false);
-
-  useEffect(() => {
-    if (!enablePushToTalk) return;
-
-    const handleKeyDown = async (event: KeyboardEvent) => {
-      if (event.code === 'Space' && !spacePressedRef.current) {
-        spacePressedRef.current = true;
-        await startRecording();
+export const VoiceRecorder = forwardRef(
+  (
+    {
+      onVoiceRecorded,
+      iconSize,
+      className,
+      disabled,
+      loading,
+      onRecordingStatusChange,
+      enablePushToTalk = false,
+    }: Props,
+    ref,
+  ) => {
+    const [isRecording, setIsRecording] = useState(false);
+    const [internalIsProcessing, setInternalIsProcessing] = useState(false);
+    const audioControllerRef = useRef<AudioController | null>(null);
+    const { ai, plugin } = useRimori();
+
+    // Ref for latest onVoiceRecorded callback
+    const onVoiceRecordedRef = useRef(onVoiceRecorded);
+    useEffect(() => {
+      onVoiceRecordedRef.current = onVoiceRecorded;
+    }, [onVoiceRecorded]);
+
+    const startRecording = async () => {
+      try {
+        if (!audioControllerRef.current) {
+          audioControllerRef.current = new AudioController(plugin.pluginId);
+        }
+
+        await audioControllerRef.current.startRecording();
+        setIsRecording(true);
+        onRecordingStatusChange(true);
+      } catch (error) {
+        console.error('Failed to start recording:', error);
+        // Handle permission denied or other errors
       }
     };
-    const handleKeyUp = (event: KeyboardEvent) => {
-      if (event.code === 'Space' && spacePressedRef.current) {
-        spacePressedRef.current = false;
-        stopRecording();
+
+    const stopRecording = async () => {
+      try {
+        if (audioControllerRef.current && isRecording) {
+          const audioResult = await audioControllerRef.current.stopRecording();
+          // console.log("audioResult: ", audioResult);
+
+          setInternalIsProcessing(true);
+
+          // Play the recorded audio from the Blob
+          // const blobUrl = URL.createObjectURL(audioResult.recording);
+          // const audioRef = new Audio(blobUrl);
+          // audioRef.onended = () => URL.revokeObjectURL(blobUrl);
+          // audioRef.play().catch((e) => console.error('Playback error:', e));
+
+          // console.log("audioBlob: ", audioResult.recording);
+          const text = await ai.getTextFromVoice(audioResult.recording);
+          // console.log("stt result", text);
+          // throw new Error("test");
+          setInternalIsProcessing(false);
+          onVoiceRecordedRef.current(text);
+        }
+      } catch (error) {
+        console.error('Failed to stop recording:', error);
+      } finally {
+        setIsRecording(false);
+        onRecordingStatusChange(false);
       }
     };
-    window.addEventListener('keydown', handleKeyDown);
-    window.addEventListener('keyup', handleKeyUp);
-    return () => {
-      window.removeEventListener('keydown', handleKeyDown);
-      window.removeEventListener('keyup', handleKeyUp);
-    };
-  }, [enablePushToTalk]);
 
-  return (
-    <button className={"flex flex-row justify-center items-center rounded-full mx-auto disabled:opacity-50 " + className}
-      onClick={isRecording ? stopRecording : startRecording}
-      disabled={disabled || loading || internalIsProcessing}>
-      {loading || internalIsProcessing ? <FaSpinner className="animate-spin" /> :
-        <FaMicrophone size={iconSize} className={(isRecording ? "text-red-600" : "")} />
-      }
-    </button>
-  );
-});
+    useImperativeHandle(ref, () => ({
+      startRecording,
+      stopRecording,
+    }));
+
+    // push to talk feature
+    const spacePressedRef = useRef(false);
+
+    useEffect(() => {
+      if (!enablePushToTalk) return;
+
+      const handleKeyDown = async (event: KeyboardEvent) => {
+        if (event.code === 'Space' && !spacePressedRef.current) {
+          spacePressedRef.current = true;
+          await startRecording();
+        }
+      };
+      const handleKeyUp = (event: KeyboardEvent) => {
+        if (event.code === 'Space' && spacePressedRef.current) {
+          spacePressedRef.current = false;
+          stopRecording();
+        }
+      };
+      window.addEventListener('keydown', handleKeyDown);
+      window.addEventListener('keyup', handleKeyUp);
+      return () => {
+        window.removeEventListener('keydown', handleKeyDown);
+        window.removeEventListener('keyup', handleKeyUp);
+      };
+    }, [enablePushToTalk]);
+
+    return (
+      <button
+        className={'flex flex-row justify-center items-center rounded-full mx-auto disabled:opacity-50 ' + className}
+        onClick={isRecording ? stopRecording : startRecording}
+        disabled={disabled || loading || internalIsProcessing}
+      >
+        {loading || internalIsProcessing ? (
+          <FaSpinner className="animate-spin" />
+        ) : (
+          <FaMicrophone size={iconSize} className={isRecording ? 'text-red-600' : ''} />
+        )}
+      </button>
+    );
+  },
+);

package/src/components/ai/utils.ts

@@ -8,14 +8,14 @@ export function getFirstMessages(instructions: FirstMessages): any[] {
   const messages = [];
 
   if (instructions.instructions) {
-      messages.push({ id: '1', role: 'system', content: instructions.instructions });
+    messages.push({ id: '1', role: 'system', content: instructions.instructions });
   }
   if (instructions.userMessage) {
-      messages.push({ id: '2', role: 'user', content: instructions.userMessage });
+    messages.push({ id: '2', role: 'user', content: instructions.userMessage });
   }
   if (instructions.assistantMessage) {
-      messages.push({ id: '3', role: 'assistant', content: instructions.assistantMessage });
+    messages.push({ id: '3', role: 'assistant', content: instructions.assistantMessage });
   }
 
   return messages;
-}
+}