@rimori/client 1.4.10 → 2.1.0

package/src/components/ai/Assistant.tsx

@@ -1,96 +0,0 @@
-import React, { useEffect, useMemo } from 'react';
-import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
-import { AudioInputField } from './EmbeddedAssistent/AudioInputField';
-import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
-import Markdown from 'react-markdown';
-import { useChat } from '../../hooks/UseChatHook';
-import { useRimori } from '../../components';
-import { FirstMessages, getFirstMessages } from './utils';
-
-interface Props {
-  voiceId: any;
-  avatarImageUrl: string;
-  onComplete: (result: any) => void;
-  autoStartConversation?: FirstMessages;
-}
-
-export function AssistantChat({ avatarImageUrl, voiceId, onComplete, autoStartConversation }: Props) {
-  const [oralCommunication, setOralCommunication] = React.useState(true);
-  const { ai: llm, event } = useRimori();
-  const sender = useMemo(() => new MessageSender(llm.getVoice, voiceId), []);
-  const { messages, append, isLoading, setMessages } = useChat();
-
-  const lastAssistantMessage = [...messages].filter((m) => m.role === 'assistant').pop()?.content;
-
-  useEffect(() => {
-    sender.setOnLoudnessChange((value: number) => event.emit('self.avatar.triggerLoudness', { loudness: value }));
-
-    if (!autoStartConversation) {
-      return;
-    }
-
-    setMessages(getFirstMessages(autoStartConversation));
-    // append([{ role: 'user', content: autoStartConversation.userMessage }]);
-
-    if (autoStartConversation.assistantMessage) {
-      // console.log("autostartmessages", { autoStartConversation, isLoading });
-      sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
-    }
-  }, []);
-
-  useEffect(() => {
-    let message = lastAssistantMessage;
-    if (message !== messages[messages.length - 1]?.content) {
-      message = undefined;
-    }
-    sender.handleNewText(message, isLoading);
-  }, [messages, isLoading]);
-
-  const lastMessage = messages[messages.length - 1];
-
-  useEffect(() => {
-    console.log('lastMessage', lastMessage);
-    const toolInvocations = lastMessage?.toolCalls;
-    if (toolInvocations && toolInvocations.length > 0) {
-      console.log('toolInvocations', toolInvocations);
-      onComplete(toolInvocations[0].args);
-    }
-  }, [lastMessage]);
-
-  if (lastMessage?.toolCalls && lastMessage.toolCalls.length > 0) {
-    console.log('lastMessage test2', lastMessage);
-    const args = lastMessage.toolCalls[0].args;
-
-    const success = args.explanationUnderstood === 'TRUE' || args.studentKnowsTopic === 'TRUE';
-
-    return (
-      <div className="px-5 pt-5 overflow-y-auto text-center" style={{ height: '478px' }}>
-        <h1 className="text-center mt-5 mb-5">{success ? 'Great job!' : 'You failed'}</h1>
-        <p>{args.improvementHints}</p>
-      </div>
-    );
-  }
-
-  return (
-    <div>
-      {oralCommunication && <CircleAudioAvatar imageUrl={avatarImageUrl} className="mx-auto my-10" />}
-      <div className="w-full">
-        {lastAssistantMessage && (
-          <div className="px-5 pt-5 overflow-y-auto remirror-theme" style={{ height: '4k78px' }}>
-            <Markdown>{lastAssistantMessage}</Markdown>
-          </div>
-        )}
-      </div>
-      <AudioInputField
-        blockSubmission={isLoading}
-        onSubmit={(message) => {
-          append([{ role: 'user', content: message, id: messages.length.toString() }]);
-        }}
-        onAudioControl={(voice) => {
-          setOralCommunication(voice);
-          sender.setVolume(voice ? 1 : 0);
-        }}
-      />
-    </div>
-  );
-}

package/src/components/ai/Avatar.tsx

@@ -1,99 +0,0 @@
-import { useEffect, useMemo, useState } from 'react';
-import { VoiceRecorder } from './EmbeddedAssistent/VoiceRecoder';
-import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
-import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
-import { Tool } from '../../fromRimori/PluginTypes';
-import { useChat } from '../../hooks/UseChatHook';
-import { useRimori } from '../../components';
-import { getFirstMessages } from './utils';
-import { FirstMessages } from './utils';
-
-interface Props {
-  voiceId: string;
-  agentTools: Tool[];
-  avatarImageUrl: string;
-  circleSize?: string;
-  isDarkTheme?: boolean;
-  children?: React.ReactNode;
-  autoStartConversation?: FirstMessages;
-  className?: string;
-}
-
-export function Avatar({
-  avatarImageUrl,
-  voiceId,
-  agentTools,
-  autoStartConversation,
-  children,
-  isDarkTheme = false,
-  circleSize = '300px',
-  className,
-}: Props) {
-  const { ai, event } = useRimori();
-  const [agentReplying, setAgentReplying] = useState(false);
-  const [isProcessingMessage, setIsProcessingMessage] = useState(false);
-  const sender = useMemo(() => new MessageSender(ai.getVoice, voiceId), [voiceId]);
-  const { messages, append, isLoading, lastMessage, setMessages } = useChat(agentTools);
-
-  useEffect(() => {
-    console.log('messages', messages);
-  }, [messages]);
-
-  useEffect(() => {
-    if (!isLoading) setIsProcessingMessage(false);
-  }, [isLoading]);
-
-  useEffect(() => {
-    if (!voiceId) return; //at the beginning when being mounted the voiceId is undefined
-    sender.setOnLoudnessChange((value) => event.emit('self.avatar.triggerLoudness', { loudness: value }));
-    sender.setOnEndOfSpeech(() => setAgentReplying(false));
-
-    if (!autoStartConversation) return;
-
-    setMessages(getFirstMessages(autoStartConversation));
-    // append([{ role: 'user', content: autoStartConversation.userMessage }]);
-
-    if (autoStartConversation.assistantMessage) {
-      // console.log("autostartmessages", { autoStartConversation, isLoading });
-      sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
-    } else if (autoStartConversation.userMessage) {
-      append([{ role: 'user', content: autoStartConversation.userMessage, id: messages.length.toString() }]);
-    }
-  }, [autoStartConversation, voiceId]);
-
-  useEffect(() => {
-    if (lastMessage?.role === 'assistant') {
-      sender.handleNewText(lastMessage.content, isLoading);
-      if (lastMessage.toolCalls) {
-        // console.log("unlocking mic", lastMessage)
-        setAgentReplying(false);
-        setIsProcessingMessage(false);
-      }
-    }
-  }, [lastMessage, isLoading]);
-
-  return (
-    <div className={`md:pb-8 ${className || ''}`}>
-      <CircleAudioAvatar width={circleSize} className="mx-auto" imageUrl={avatarImageUrl} isDarkTheme={isDarkTheme} />
-      {children}
-      <VoiceRecorder
-        iconSize="30"
-        className="w-16 h-16 shadow-lg rounded-full bg-gray-400 dark:bg-gray-800"
-        disabled={agentReplying}
-        loading={isProcessingMessage}
-        enablePushToTalk={true}
-        onVoiceRecorded={(message) => {
-          setAgentReplying(true);
-          append([
-            {
-              role: 'user',
-              content: 'Message(' + Math.floor((messages.length + 1) / 2) + '): ' + message,
-              id: messages.length.toString(),
-            },
-          ]);
-        }}
-        onRecordingStatusChange={(running) => !running && setIsProcessingMessage(true)}
-      />
-    </div>
-  );
-}

package/src/components/ai/EmbeddedAssistent/AudioInputField.tsx

@@ -1,73 +0,0 @@
-import React, { useState } from 'react';
-import { VoiceRecorder } from './VoiceRecoder';
-import { BiSolidRightArrow } from 'react-icons/bi';
-import { HiMiniSpeakerXMark, HiMiniSpeakerWave } from 'react-icons/hi2';
-
-interface AudioInputFieldProps {
-  onSubmit: (text: string) => void;
-  onAudioControl?: (voice: boolean) => void;
-  blockSubmission?: boolean;
-}
-
-export function AudioInputField({ onSubmit, onAudioControl, blockSubmission = false }: AudioInputFieldProps) {
-  const [text, setText] = useState('');
-  const [audioEnabled, setAudioEnabled] = useState(true);
-
-  const handleSubmit = (manualText?: string) => {
-    if (blockSubmission) return;
-    const sendableText = manualText || text;
-    if (sendableText.trim()) {
-      onSubmit(sendableText);
-      setTimeout(() => {
-        setText('');
-      }, 100);
-    }
-  };
-
-  const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
-    if (blockSubmission) return;
-    if (e.key === 'Enter' && e.ctrlKey) {
-      setText(text + '\n');
-    } else if (e.key === 'Enter') {
-      handleSubmit();
-    }
-  };
-
-  return (
-    <div className="flex items-center bg-gray-600 pt-2 pb-2 p-2">
-      {onAudioControl && (
-        <button
-          onClick={() => {
-            onAudioControl(!audioEnabled);
-            setAudioEnabled(!audioEnabled);
-          }}
-          className="cursor-default"
-        >
-          {audioEnabled ? (
-            <HiMiniSpeakerWave className="w-9 h-9 cursor-pointer" />
-          ) : (
-            <HiMiniSpeakerXMark className="w-9 h-9 cursor-pointer" />
-          )}
-        </button>
-      )}
-      <VoiceRecorder
-        onRecordingStatusChange={() => {}}
-        onVoiceRecorded={(m: string) => {
-          console.log('onVoiceRecorded', m);
-          handleSubmit(m);
-        }}
-      />
-      <textarea
-        value={text}
-        onChange={(e) => setText(e.target.value)}
-        onKeyDown={handleKeyDown}
-        className="flex-1 border-none rounded-lg p-2 text-gray-800 focus::outline-none"
-        placeholder="Type a message..."
-        disabled={blockSubmission}
-      />
-      <button onClick={() => handleSubmit()} className="cursor-default" disabled={blockSubmission}>
-        <BiSolidRightArrow className="w-9 h-10 cursor-pointer" />
-      </button>
-    </div>
-  );
-}

package/src/components/ai/EmbeddedAssistent/CircleAudioAvatar.tsx

@@ -1,107 +0,0 @@
-import { useEffect, useRef } from 'react';
-import { EventBus, EventBusMessage } from '../../../fromRimori/EventBus';
-
-interface CircleAudioAvatarProps {
-  width?: string;
-  imageUrl: string;
-  className?: string;
-  isDarkTheme?: boolean;
-}
-
-export function CircleAudioAvatar({
-  imageUrl,
-  className,
-  isDarkTheme = false,
-  width = '150px',
-}: CircleAudioAvatarProps) {
-  const canvasRef = useRef<HTMLCanvasElement>(null);
-  const currentLoudnessRef = useRef(0);
-  const targetLoudnessRef = useRef(0);
-  const animationFrameRef = useRef<number | null>(null);
-
-  useEffect(() => {
-    const canvas = canvasRef.current;
-    if (canvas) {
-      const ctx = canvas.getContext('2d');
-      if (ctx) {
-        const image = new Image();
-        image.src = imageUrl;
-        let isMounted = true;
-
-        image.onload = () => {
-          if (!isMounted) return;
-          draw(ctx, canvas, image, 0);
-          const animate = () => {
-            const decayRate = 0.06;
-            if (currentLoudnessRef.current > targetLoudnessRef.current) {
-              currentLoudnessRef.current = Math.max(
-                targetLoudnessRef.current,
-                currentLoudnessRef.current - decayRate * currentLoudnessRef.current,
-              );
-            } else {
-              currentLoudnessRef.current = targetLoudnessRef.current;
-            }
-            draw(ctx, canvas, image, currentLoudnessRef.current);
-            animationFrameRef.current = requestAnimationFrame(animate);
-          };
-          animationFrameRef.current = requestAnimationFrame(animate);
-        };
-
-        const handleLoudness = ({ data }: EventBusMessage) => {
-          const newLoudness = data.loudness;
-          if (newLoudness > currentLoudnessRef.current) {
-            currentLoudnessRef.current = newLoudness;
-          }
-          targetLoudnessRef.current = newLoudness;
-        };
-
-        const listener = EventBus.on('self.avatar.triggerLoudness', handleLoudness);
-
-        return () => {
-          isMounted = false;
-          listener.off();
-          if (animationFrameRef.current) {
-            cancelAnimationFrame(animationFrameRef.current);
-          }
-        };
-      }
-    }
-  }, [imageUrl]);
-
-  const draw = (
-    ctx: CanvasRenderingContext2D,
-    canvas: HTMLCanvasElement,
-    image: HTMLImageElement,
-    loudness: number,
-  ) => {
-    if (canvas && ctx) {
-      ctx.clearRect(0, 0, canvas.width, canvas.height);
-
-      const radius = Math.min(canvas.width, canvas.height) / 3;
-      const centerX = canvas.width / 2;
-      const centerY = canvas.height / 2;
-      const pulseRadius = radius + loudness / 2.5;
-      ctx.beginPath();
-      ctx.arc(centerX, centerY, pulseRadius, 0, Math.PI * 2, true);
-      ctx.strokeStyle = isDarkTheme ? 'rgba(255, 255, 255, 0.5)' : 'rgba(0, 0, 0, 0.5)';
-      ctx.lineWidth = 5;
-      ctx.stroke();
-
-      ctx.save();
-      ctx.beginPath();
-      ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
-      ctx.closePath();
-      ctx.clip();
-      ctx.drawImage(image, centerX - radius, centerY - radius, radius * 2, radius * 2);
-      ctx.restore();
-
-      ctx.beginPath();
-      ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
-      ctx.strokeStyle = isDarkTheme ? 'rgba(255, 255, 255, 0.9)' : 'rgba(0, 0, 0, 0.9)';
-      ctx.lineWidth = 5;
-      ctx.stroke();
-    }
-  };
-
-  return <canvas ref={canvasRef} className={className} width={500} height={500} style={{ width }} />;
-}

package/src/components/ai/EmbeddedAssistent/TTS/MessageSender.ts

@@ -1,96 +0,0 @@
-import { ChunkedAudioPlayer } from './Player';
-
-type VoiceBackend = (text: string, voice?: string, speed?: number) => Promise<Blob>;
-
-export class MessageSender {
-  private player = new ChunkedAudioPlayer();
-  private fetchedSentences = new Set<string>();
-  private lastLoading = false;
-  private voice: string;
-  private voiceBackend: VoiceBackend;
-
-  constructor(voiceBackend: VoiceBackend, voice: string) {
-    if (voice?.split('_').length !== 2) {
-      throw new Error("Invalid voice id format '" + voice + "'. Voice id needs to look like <provider>_<voice_id>");
-    }
-    this.voiceBackend = voiceBackend;
-    this.voice = voice;
-  }
-
-  private getCompletedSentences(currentText: string, isLoading: boolean): string[] {
-    // Split the text based on the following characters: .?!
-    // Only split on : when followed by a space
-    const pattern = /(.+?[.?!]|.+?:\s+|.+?\n+)/g;
-    const result: string[] = [];
-    let match;
-    while ((match = pattern.exec(currentText)) !== null) {
-      const sentence = match[0].trim();
-      if (sentence.length > 0) {
-        result.push(sentence);
-      }
-    }
-    if (!isLoading) {
-      const lastFullSentence = result[result.length - 1];
-      const leftoverIndex = currentText.lastIndexOf(lastFullSentence) + lastFullSentence.length;
-      if (leftoverIndex < currentText.length) {
-        result.push(currentText.slice(leftoverIndex).trim());
-      }
-    }
-    return result;
-  }
-
-  public async handleNewText(currentText: string | undefined, isLoading: boolean) {
-    if (!this.lastLoading && isLoading) {
-      this.reset();
-    }
-    this.lastLoading = isLoading;
-
-    if (!currentText) {
-      return;
-    }
-
-    const sentences = this.getCompletedSentences(currentText, isLoading);
-
-    for (let i = 0; i < sentences.length; i++) {
-      const sentence = sentences[i];
-      if (!this.fetchedSentences.has(sentence)) {
-        this.fetchedSentences.add(sentence);
-        const audioData = await this.generateSpeech(sentence);
-        await this.player.addChunk(audioData, i);
-      }
-    }
-  }
-
-  private async generateSpeech(sentence: string): Promise<ArrayBuffer> {
-    const blob = await this.voiceBackend(sentence, this.voice, 1.0);
-    return await blob.arrayBuffer();
-  }
-
-  public play() {
-    this.player.playAgain();
-  }
-
-  public stop() {
-    this.player.stopPlayback();
-  }
-
-  private reset() {
-    this.stop();
-    this.fetchedSentences.clear();
-    this.player.reset();
-  }
-
-  public setVolume(volume: number) {
-    this.player.setVolume(volume);
-  }
-
-  public setOnLoudnessChange(callback: (value: number) => void) {
-    this.player.setOnLoudnessChange((loudness) => {
-      callback(loudness);
-    });
-  }
-
-  public setOnEndOfSpeech(callback: () => void) {
-    this.player.setOnEndOfSpeech(callback);
-  }
-}

package/src/components/ai/EmbeddedAssistent/TTS/Player.ts

@@ -1,197 +0,0 @@
-export class ChunkedAudioPlayer {
-  private audioContext!: AudioContext;
-  private chunkQueue: ArrayBuffer[] = [];
-  private isPlaying = false;
-  private analyser!: AnalyserNode;
-  private dataArray!: Uint8Array<ArrayBuffer>;
-  private shouldMonitorLoudness = true;
-  private isMonitoring = false;
-  private handle = 0;
-  private volume = 1.0;
-  private loudnessCallback: (value: number) => void = () => {};
-  private currentIndex = 0;
-  private startedPlaying = false;
-  private onEndOfSpeech: () => void = () => {};
-
-  constructor() {
-    this.init();
-  }
-
-  private init(): void {
-    this.audioContext = new AudioContext();
-    this.analyser = this.audioContext.createAnalyser();
-    this.analyser.fftSize = 256; // Set the FFT size (smaller values provide faster updates, larger ones give better resolution)
-    const bufferLength = this.analyser.frequencyBinCount;
-    this.dataArray = new Uint8Array(bufferLength); // Array to hold frequency data
-  }
-
-  public setOnLoudnessChange(callback: (value: number) => void) {
-    this.loudnessCallback = callback;
-  }
-
-  public setVolume(volume: number) {
-    this.volume = volume;
-  }
-
-  public async addChunk(chunk: ArrayBuffer, position: number): Promise<void> {
-    console.log('Adding chunk', position, chunk);
-    this.chunkQueue[position] = chunk;
-    // console.log("received chunk", {
-    //     chunkQueue: this.chunkQueue.length,
-    //     isPlaying: this.isPlaying,
-    // })
-
-    if (position === 0 && !this.startedPlaying) {
-      this.startedPlaying = true;
-      this.playChunks();
-    }
-  }
-
-  private playChunks(): void {
-    // console.log({ isPlaying: this.isPlaying });
-    if (this.isPlaying) return;
-    if (!this.chunkQueue[this.currentIndex]) {
-      // wait until the correct chunk arrives
-      setTimeout(() => this.playChunks(), 10);
-    }
-    this.isPlaying = true;
-
-    this.playChunk(this.chunkQueue[this.currentIndex]).then(() => {
-      this.isPlaying = false;
-      this.currentIndex++;
-      if (this.chunkQueue[this.currentIndex]) {
-        this.shouldMonitorLoudness = true;
-        this.playChunks();
-      } else {
-        // console.log('Playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
-        setTimeout(() => {
-          // console.log('Check again if really playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
-          if (this.chunkQueue.length > this.currentIndex) {
-            this.playChunks();
-          } else {
-            this.startedPlaying = false;
-            this.shouldMonitorLoudness = false;
-          }
-        }, 1000);
-      }
-    });
-  }
-
-  public stopPlayback(): void {
-    // console.log('Stopping playback');
-    // Implement logic to stop the current playback
-    this.isPlaying = false;
-    this.chunkQueue = [];
-    this.startedPlaying = false;
-    this.shouldMonitorLoudness = false;
-    cancelAnimationFrame(this.handle);
-  }
-
-  private playChunk(chunk: ArrayBuffer): Promise<void> {
-    // console.log({queue: this.chunkQueue})
-    if (!chunk) {
-      return Promise.resolve();
-    }
-
-    // console.log('Playing chunk', chunk);
-    return new Promise((resolve) => {
-      const source = this.audioContext.createBufferSource();
-      this.audioContext.decodeAudioData(chunk.slice(0)).then((audioBuffer) => {
-        source.buffer = audioBuffer;
-
-        // Create a GainNode for volume control
-        const gainNode = this.audioContext.createGain();
-        gainNode.gain.value = this.volume;
-
-        // Connect the source to the GainNode, then to the analyser node, then to the destination (speakers)
-        source.connect(gainNode);
-        gainNode.connect(this.analyser);
-        this.analyser.connect(this.audioContext.destination);
-
-        source.start(0);
-        // console.log('Playing chunk', this.currentIndex);
-        gainNode.gain.value = this.volume;
-        source.onended = () => {
-          // console.log('Chunk ended');
-          resolve();
-        };
-
-        // Start monitoring loudness only once
-        if (!this.isMonitoring) {
-          this.isMonitoring = true;
-          this.shouldMonitorLoudness = true;
-          this.monitorLoudness();
-        }
-      });
-    });
-  }
-
-  async playAgain(): Promise<void> {
-    console.log('Playing again');
-    if (this.chunkQueue.length > 0 && !this.isPlaying) {
-      this.playChunks();
-    }
-  }
-
-  private monitorLoudness(): void {
-    // Stop monitoring when the flag is false
-    if (!this.shouldMonitorLoudness) {
-      // console.log('Loudness monitoring stopped.');
-      cancelAnimationFrame(this.handle);
-      this.loudnessCallback(0);
-      this.onEndOfSpeech();
-      return;
-    }
-
-    // Get the time domain data from the analyser (this is a snapshot of the waveform)
-    this.analyser.getByteTimeDomainData(this.dataArray);
-
-    // Calculate the RMS (root mean square) of the waveform values to get the perceived loudness
-    let sum = 0;
-    for (let i = 0; i < this.dataArray.length; i++) {
-      const value = this.dataArray[i] / 128.0 - 1.0; // Normalize between -1 and 1
-      sum += value * value;
-    }
-
-    const rms = Math.sqrt(sum / this.dataArray.length);
-
-    // Handle the case where RMS is 0 to avoid log10(0)
-    if (rms === 0) {
-      // console.log('Current loudness: Silent');
-    } else {
-      let loudnessInDb = 20 * Math.log10(rms); // Convert to dB
-      // console.log('Current loudness:' + loudnessInDb);
-      const minDb = -57;
-      const maxDb = -15;
-
-      if (loudnessInDb < minDb) {
-        loudnessInDb = minDb;
-      }
-      if (loudnessInDb > maxDb) {
-        loudnessInDb = maxDb;
-      }
-
-      const loudnessScale = ((loudnessInDb - minDb) / (maxDb - minDb)) * 100;
-      // console.log("root:corrent loudness", loudnessScale);
-
-      this.loudnessCallback(loudnessScale);
-    }
-
-    // Call this method again at regular intervals if you want continuous loudness monitoring
-    this.handle = requestAnimationFrame(() => this.monitorLoudness());
-  }
-  public reset() {
-    // console.log('Resetting player');
-    this.stopPlayback();
-    this.currentIndex = 0;
-    this.shouldMonitorLoudness = true;
-    //reset to the beginning when the class gets initialized
-    this.isMonitoring = false;
-    this.isPlaying = false;
-    this.init();
-  }
-
-  public setOnEndOfSpeech(callback: () => void) {
-    this.onEndOfSpeech = callback;
-  }
-}