@rimori/react-client 0.1.0

package/src/components/MarkdownEditor.tsx

@@ -0,0 +1,144 @@
+import { Markdown } from 'tiptap-markdown';
+import StarterKit from '@tiptap/starter-kit';
+import { PiCodeBlock } from 'react-icons/pi';
+import { TbBlockquote } from 'react-icons/tb';
+import { GoListOrdered } from 'react-icons/go';
+import { AiOutlineUnorderedList } from 'react-icons/ai';
+import { EditorProvider, useCurrentEditor } from '@tiptap/react';
+import { LuHeading1, LuHeading2, LuHeading3 } from 'react-icons/lu';
+import { FaBold, FaCode, FaItalic, FaParagraph, FaStrikethrough } from 'react-icons/fa';
+
+// This inplementation is rooted in the Tiptap editor basic example https://codesandbox.io/p/devbox/editor-9x9dkd
+
+interface EditorButtonProps {
+  action: string;
+  isActive?: boolean;
+  label: string | React.ReactNode;
+  disabled?: boolean;
+}
+
+const EditorButton = ({ action, isActive, label, disabled }: EditorButtonProps) => {
+  const { editor } = useCurrentEditor() as any;
+
+  if (!editor) {
+    return null;
+  }
+
+  if (action.includes('heading')) {
+    const level = parseInt(action[action.length - 1]);
+    return (
+      <button
+        onClick={() => editor.chain().focus().toggleHeading({ level: level }).run()}
+        className={`pl-2 ${isActive ? 'is-active' : ''}`}
+      >
+        {label}
+      </button>
+    );
+  }
+
+  return (
+    <button
+      onClick={() => editor.chain().focus()[action]().run()}
+      disabled={disabled ? !editor.can().chain().focus()[action]().run() : false}
+      className={`pl-2 ${isActive ? 'is-active' : ''}`}
+    >
+      {label}
+    </button>
+  );
+};
+
+const MenuBar = () => {
+  const { editor } = useCurrentEditor();
+
+  if (!editor) {
+    return null;
+  }
+
+  return (
+    <div className="bg-gray-400 dark:bg-gray-800 dark:text-white text-lg flex flex-row flex-wrap items-center p-1">
+      <EditorButton action="toggleBold" isActive={editor.isActive('bold')} label={<FaBold />} disabled />
+      <EditorButton action="toggleItalic" isActive={editor.isActive('italic')} label={<FaItalic />} disabled />
+      <EditorButton action="toggleStrike" isActive={editor.isActive('strike')} label={<FaStrikethrough />} disabled />
+      <EditorButton action="toggleCode" isActive={editor.isActive('code')} label={<FaCode />} disabled />
+      <EditorButton action="setParagraph" isActive={editor.isActive('paragraph')} label={<FaParagraph />} />
+      <EditorButton
+        action="setHeading1"
+        isActive={editor.isActive('heading', { level: 1 })}
+        label={<LuHeading1 size={'24px'} />}
+      />
+      <EditorButton
+        action="setHeading2"
+        isActive={editor.isActive('heading', { level: 2 })}
+        label={<LuHeading2 size={'24px'} />}
+      />
+      <EditorButton
+        action="setHeading3"
+        isActive={editor.isActive('heading', { level: 3 })}
+        label={<LuHeading3 size={'24px'} />}
+      />
+      <EditorButton
+        action="toggleBulletList"
+        isActive={editor.isActive('bulletList')}
+        label={<AiOutlineUnorderedList size={'24px'} />}
+      />
+      <EditorButton
+        action="toggleOrderedList"
+        isActive={editor.isActive('orderedList')}
+        label={<GoListOrdered size={'24px'} />}
+      />
+      <EditorButton
+        action="toggleCodeBlock"
+        isActive={editor.isActive('codeBlock')}
+        label={<PiCodeBlock size={'24px'} />}
+      />
+      <EditorButton
+        action="toggleBlockquote"
+        isActive={editor.isActive('blockquote')}
+        label={<TbBlockquote size={'24px'} />}
+      />
+    </div>
+  );
+};
+
+const extensions = [
+  StarterKit.configure({
+    bulletList: {
+      HTMLAttributes: {
+        class: 'list-disc list-inside dark:text-white p-1 mt-1 [&_li]:mb-1 [&_p]:inline m-0',
+      },
+    },
+    orderedList: {
+      HTMLAttributes: {
+        className: 'list-decimal list-inside dark:text-white p-1 mt-1 [&_li]:mb-1 [&_p]:inline m-0',
+      },
+    },
+  }),
+  Markdown,
+];
+
+interface Props {
+  content?: string;
+  editable: boolean;
+  className?: string;
+  onUpdate?: (content: string) => void;
+}
+
+export const MarkdownEditor = (props: Props) => {
+  return (
+    <div
+      className={'text-md border border-gray-800 overflow-hidden ' + props.className}
+      style={{ borderWidth: props.editable ? 1 : 0 }}
+    >
+      <EditorProvider
+        key={(props.editable ? 'editable' : 'readonly') + props.content}
+        slotBefore={props.editable ? <MenuBar /> : null}
+        extensions={extensions}
+        content={props.content}
+        editable={props.editable}
+        onUpdate={(e) => {
+          props.onUpdate && props.onUpdate(e.editor.storage.markdown.getMarkdown());
+        }}
+      ></EditorProvider>
+    </div>
+  );
+};

package/src/components/Spinner.tsx

@@ -0,0 +1,29 @@
+import React from 'react';
+
+interface SpinnerProps {
+  text?: string;
+  size?: string;
+  className?: string;
+}
+
+export const Spinner: React.FC<SpinnerProps> = ({ text, className, size = '30px' }) => {
+  return (
+    <div className={'flex items-center space-x-2 ' + className}>
+      <svg
+        style={{ width: size, height: size }}
+        className="animate-spin -ml-1 mr-3 h-5 w-5 text-white"
+        xmlns="http://www.w3.org/2000/svg"
+        fill="none"
+        viewBox="0 0 24 24"
+      >
+        <circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
+        <path
+          className="opacity-75"
+          fill="currentColor"
+          d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
+        ></path>
+      </svg>
+      {text && <span className="">{text}</span>}
+    </div>
+  );
+};

package/src/components/ai/Assistant.tsx

@@ -0,0 +1,96 @@
+import React, { useEffect, useMemo } from 'react';
+import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
+import { AudioInputField } from './EmbeddedAssistent/AudioInputField';
+import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
+import Markdown from 'react-markdown';
+import { useChat } from '../../hooks/UseChatHook';
+import { useRimori } from '../../providers/PluginProvider';
+import { FirstMessages, getFirstMessages } from './utils';
+
+interface Props {
+  voiceId: any;
+  avatarImageUrl: string;
+  onComplete: (result: any) => void;
+  autoStartConversation?: FirstMessages;
+}
+
+export function AssistantChat({ avatarImageUrl, voiceId, onComplete, autoStartConversation }: Props) {
+  const [oralCommunication, setOralCommunication] = React.useState(true);
+  const { ai: llm, event } = useRimori();
+  const sender = useMemo(() => new MessageSender(llm.getVoice, voiceId), []);
+  const { messages, append, isLoading, setMessages } = useChat();
+
+  const lastAssistantMessage = [...messages].filter((m) => m.role === 'assistant').pop()?.content;
+
+  useEffect(() => {
+    sender.setOnLoudnessChange((value: number) => event.emit('self.avatar.triggerLoudness', { loudness: value }));
+
+    if (!autoStartConversation) {
+      return;
+    }
+
+    setMessages(getFirstMessages(autoStartConversation));
+    // append([{ role: 'user', content: autoStartConversation.userMessage }]);
+
+    if (autoStartConversation.assistantMessage) {
+      // console.log("autostartmessages", { autoStartConversation, isLoading });
+      sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
+    }
+  }, []);
+
+  useEffect(() => {
+    let message = lastAssistantMessage;
+    if (message !== messages[messages.length - 1]?.content) {
+      message = undefined;
+    }
+    sender.handleNewText(message, isLoading);
+  }, [messages, isLoading]);
+
+  const lastMessage = messages[messages.length - 1];
+
+  useEffect(() => {
+    console.log('lastMessage', lastMessage);
+    const toolInvocations = lastMessage?.toolCalls;
+    if (toolInvocations && toolInvocations.length > 0) {
+      console.log('toolInvocations', toolInvocations);
+      onComplete(toolInvocations[0].args);
+    }
+  }, [lastMessage]);
+
+  if (lastMessage?.toolCalls && lastMessage.toolCalls.length > 0) {
+    console.log('lastMessage test2', lastMessage);
+    const args = lastMessage.toolCalls[0].args;
+
+    const success = args.explanationUnderstood === 'TRUE' || args.studentKnowsTopic === 'TRUE';
+
+    return (
+      <div className="px-5 pt-5 overflow-y-auto text-center" style={{ height: '478px' }}>
+        <h1 className="text-center mt-5 mb-5">{success ? 'Great job!' : 'You failed'}</h1>
+        <p>{args.improvementHints}</p>
+      </div>
+    );
+  }
+
+  return (
+    <div>
+      {oralCommunication && <CircleAudioAvatar imageUrl={avatarImageUrl} className="mx-auto my-10" />}
+      <div className="w-full">
+        {lastAssistantMessage && (
+          <div className="px-5 pt-5 overflow-y-auto remirror-theme" style={{ height: '4k78px' }}>
+            <Markdown>{lastAssistantMessage}</Markdown>
+          </div>
+        )}
+      </div>
+      <AudioInputField
+        blockSubmission={isLoading}
+        onSubmit={(message) => {
+          append([{ role: 'user', content: message, id: messages.length.toString() }]);
+        }}
+        onAudioControl={(voice) => {
+          setOralCommunication(voice);
+          sender.setVolume(voice ? 1 : 0);
+        }}
+      />
+    </div>
+  );
+}

package/src/components/ai/Avatar.tsx

@@ -0,0 +1,99 @@
+import { useEffect, useMemo, useState } from 'react';
+import { VoiceRecorder } from './EmbeddedAssistent/VoiceRecoder';
+import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
+import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
+import { Tool } from '@rimori/client';
+import { useChat } from '../../hooks/UseChatHook';
+import { useRimori } from '../../providers/PluginProvider';
+import { getFirstMessages } from './utils';
+import { FirstMessages } from './utils';
+
+interface Props {
+  voiceId: string;
+  agentTools: Tool[];
+  avatarImageUrl: string;
+  circleSize?: string;
+  isDarkTheme?: boolean;
+  children?: React.ReactNode;
+  autoStartConversation?: FirstMessages;
+  className?: string;
+}
+
+export function Avatar({
+  avatarImageUrl,
+  voiceId,
+  agentTools,
+  autoStartConversation,
+  children,
+  isDarkTheme = false,
+  circleSize = '300px',
+  className,
+}: Props) {
+  const { ai, event } = useRimori();
+  const [agentReplying, setAgentReplying] = useState(false);
+  const [isProcessingMessage, setIsProcessingMessage] = useState(false);
+  const sender = useMemo(() => new MessageSender(ai.getVoice, voiceId), [voiceId]);
+  const { messages, append, isLoading, lastMessage, setMessages } = useChat(agentTools);
+
+  useEffect(() => {
+    console.log('messages', messages);
+  }, [messages]);
+
+  useEffect(() => {
+    if (!isLoading) setIsProcessingMessage(false);
+  }, [isLoading]);
+
+  useEffect(() => {
+    if (!voiceId) return; //at the beginning when being mounted the voiceId is undefined
+    sender.setOnLoudnessChange((value) => event.emit('self.avatar.triggerLoudness', { loudness: value }));
+    sender.setOnEndOfSpeech(() => setAgentReplying(false));
+
+    if (!autoStartConversation) return;
+
+    setMessages(getFirstMessages(autoStartConversation));
+    // append([{ role: 'user', content: autoStartConversation.userMessage }]);
+
+    if (autoStartConversation.assistantMessage) {
+      // console.log("autostartmessages", { autoStartConversation, isLoading });
+      sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
+    } else if (autoStartConversation.userMessage) {
+      append([{ role: 'user', content: autoStartConversation.userMessage, id: messages.length.toString() }]);
+    }
+  }, [autoStartConversation, voiceId]);
+
+  useEffect(() => {
+    if (lastMessage?.role === 'assistant') {
+      sender.handleNewText(lastMessage.content, isLoading);
+      if (lastMessage.toolCalls) {
+        // console.log("unlocking mic", lastMessage)
+        setAgentReplying(false);
+        setIsProcessingMessage(false);
+      }
+    }
+  }, [lastMessage, isLoading]);
+
+  return (
+    <div className={`md:pb-8 ${className || ''}`}>
+      <CircleAudioAvatar width={circleSize} className="mx-auto" imageUrl={avatarImageUrl} isDarkTheme={isDarkTheme} />
+      {children}
+      <VoiceRecorder
+        iconSize="30"
+        className="w-16 h-16 shadow-lg rounded-full bg-gray-400 dark:bg-gray-800"
+        disabled={agentReplying}
+        loading={isProcessingMessage}
+        enablePushToTalk={true}
+        onVoiceRecorded={(message) => {
+          setAgentReplying(true);
+          append([
+            {
+              role: 'user',
+              content: 'Message(' + Math.floor((messages.length + 1) / 2) + '): ' + message,
+              id: messages.length.toString(),
+            },
+          ]);
+        }}
+        onRecordingStatusChange={(running) => !running && setIsProcessingMessage(true)}
+      />
+    </div>
+  );
+}

package/src/components/ai/EmbeddedAssistent/AudioInputField.tsx

@@ -0,0 +1,73 @@
+import React, { useState } from 'react';
+import { VoiceRecorder } from './VoiceRecoder';
+import { BiSolidRightArrow } from 'react-icons/bi';
+import { HiMiniSpeakerXMark, HiMiniSpeakerWave } from 'react-icons/hi2';
+
+interface AudioInputFieldProps {
+  onSubmit: (text: string) => void;
+  onAudioControl?: (voice: boolean) => void;
+  blockSubmission?: boolean;
+}
+
+export function AudioInputField({ onSubmit, onAudioControl, blockSubmission = false }: AudioInputFieldProps) {
+  const [text, setText] = useState('');
+  const [audioEnabled, setAudioEnabled] = useState(true);
+
+  const handleSubmit = (manualText?: string) => {
+    if (blockSubmission) return;
+    const sendableText = manualText || text;
+    if (sendableText.trim()) {
+      onSubmit(sendableText);
+      setTimeout(() => {
+        setText('');
+      }, 100);
+    }
+  };
+
+  const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
+    if (blockSubmission) return;
+    if (e.key === 'Enter' && e.ctrlKey) {
+      setText(text + '\n');
+    } else if (e.key === 'Enter') {
+      handleSubmit();
+    }
+  };
+
+  return (
+    <div className="flex items-center bg-gray-600 pt-2 pb-2 p-2">
+      {onAudioControl && (
+        <button
+          onClick={() => {
+            onAudioControl(!audioEnabled);
+            setAudioEnabled(!audioEnabled);
+          }}
+          className="cursor-default"
+        >
+          {audioEnabled ? (
+            <HiMiniSpeakerWave className="w-9 h-9 cursor-pointer" />
+          ) : (
+            <HiMiniSpeakerXMark className="w-9 h-9 cursor-pointer" />
+          )}
+        </button>
+      )}
+      <VoiceRecorder
+        onRecordingStatusChange={() => {}}
+        onVoiceRecorded={(m: string) => {
+          console.log('onVoiceRecorded', m);
+          handleSubmit(m);
+        }}
+      />
+      <textarea
+        value={text}
+        onChange={(e) => setText(e.target.value)}
+        onKeyDown={handleKeyDown}
+        className="flex-1 border-none rounded-lg p-2 text-gray-800 focus::outline-none"
+        placeholder="Type a message..."
+        disabled={blockSubmission}
+      />
+      <button onClick={() => handleSubmit()} className="cursor-default" disabled={blockSubmission}>
+        <BiSolidRightArrow className="w-9 h-10 cursor-pointer" />
+      </button>
+    </div>
+  );
+}

package/src/components/ai/EmbeddedAssistent/CircleAudioAvatar.tsx

@@ -0,0 +1,107 @@
+import { useEffect, useRef } from 'react';
+import { EventBus, EventBusMessage } from '@rimori/client';
+
+interface CircleAudioAvatarProps {
+  width?: string;
+  imageUrl: string;
+  className?: string;
+  isDarkTheme?: boolean;
+}
+
+export function CircleAudioAvatar({
+  imageUrl,
+  className,
+  isDarkTheme = false,
+  width = '150px',
+}: CircleAudioAvatarProps) {
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+  const currentLoudnessRef = useRef(0);
+  const targetLoudnessRef = useRef(0);
+  const animationFrameRef = useRef<number | null>(null);
+
+  useEffect(() => {
+    const canvas = canvasRef.current;
+    if (canvas) {
+      const ctx = canvas.getContext('2d');
+      if (ctx) {
+        const image = new Image();
+        image.src = imageUrl;
+        let isMounted = true;
+
+        image.onload = () => {
+          if (!isMounted) return;
+          draw(ctx, canvas, image, 0);
+          const animate = () => {
+            const decayRate = 0.06;
+            if (currentLoudnessRef.current > targetLoudnessRef.current) {
+              currentLoudnessRef.current = Math.max(
+                targetLoudnessRef.current,
+                currentLoudnessRef.current - decayRate * currentLoudnessRef.current,
+              );
+            } else {
+              currentLoudnessRef.current = targetLoudnessRef.current;
+            }
+            draw(ctx, canvas, image, currentLoudnessRef.current);
+            animationFrameRef.current = requestAnimationFrame(animate);
+          };
+          animationFrameRef.current = requestAnimationFrame(animate);
+        };
+
+        const handleLoudness = ({ data }: EventBusMessage) => {
+          const newLoudness = data.loudness;
+          if (newLoudness > currentLoudnessRef.current) {
+            currentLoudnessRef.current = newLoudness;
+          }
+          targetLoudnessRef.current = newLoudness;
+        };
+
+        const listener = EventBus.on('self.avatar.triggerLoudness', handleLoudness);
+
+        return () => {
+          isMounted = false;
+          listener.off();
+          if (animationFrameRef.current) {
+            cancelAnimationFrame(animationFrameRef.current);
+          }
+        };
+      }
+    }
+  }, [imageUrl]);
+
+  const draw = (
+    ctx: CanvasRenderingContext2D,
+    canvas: HTMLCanvasElement,
+    image: HTMLImageElement,
+    loudness: number,
+  ) => {
+    if (canvas && ctx) {
+      ctx.clearRect(0, 0, canvas.width, canvas.height);
+
+      const radius = Math.min(canvas.width, canvas.height) / 3;
+      const centerX = canvas.width / 2;
+      const centerY = canvas.height / 2;
+      const pulseRadius = radius + loudness / 2.5;
+      ctx.beginPath();
+      ctx.arc(centerX, centerY, pulseRadius, 0, Math.PI * 2, true);
+      ctx.strokeStyle = isDarkTheme ? 'rgba(255, 255, 255, 0.5)' : 'rgba(0, 0, 0, 0.5)';
+      ctx.lineWidth = 5;
+      ctx.stroke();
+
+      ctx.save();
+      ctx.beginPath();
+      ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
+      ctx.closePath();
+      ctx.clip();
+      ctx.drawImage(image, centerX - radius, centerY - radius, radius * 2, radius * 2);
+      ctx.restore();
+
+      ctx.beginPath();
+      ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
+      ctx.strokeStyle = isDarkTheme ? 'rgba(255, 255, 255, 0.9)' : 'rgba(0, 0, 0, 0.9)';
+      ctx.lineWidth = 5;
+      ctx.stroke();
+    }
+  };
+
+  return <canvas ref={canvasRef} className={className} width={500} height={500} style={{ width }} />;
+}

package/src/components/ai/EmbeddedAssistent/TTS/MessageSender.ts

@@ -0,0 +1,96 @@
+import { ChunkedAudioPlayer } from './Player';
+
+type VoiceBackend = (text: string, voice?: string, speed?: number) => Promise<Blob>;
+
+export class MessageSender {
+  private player = new ChunkedAudioPlayer();
+  private fetchedSentences = new Set<string>();
+  private lastLoading = false;
+  private voice: string;
+  private voiceBackend: VoiceBackend;
+
+  constructor(voiceBackend: VoiceBackend, voice: string) {
+    if (voice?.split('_').length !== 2) {
+      throw new Error("Invalid voice id format '" + voice + "'. Voice id needs to look like <provider>_<voice_id>");
+    }
+    this.voiceBackend = voiceBackend;
+    this.voice = voice;
+  }
+
+  private getCompletedSentences(currentText: string, isLoading: boolean): string[] {
+    // Split the text based on the following characters: .?!
+    // Only split on : when followed by a space
+    const pattern = /(.+?[.?!]|.+?:\s+|.+?\n+)/g;
+    const result: string[] = [];
+    let match;
+    while ((match = pattern.exec(currentText)) !== null) {
+      const sentence = match[0].trim();
+      if (sentence.length > 0) {
+        result.push(sentence);
+      }
+    }
+    if (!isLoading) {
+      const lastFullSentence = result[result.length - 1];
+      const leftoverIndex = currentText.lastIndexOf(lastFullSentence) + lastFullSentence.length;
+      if (leftoverIndex < currentText.length) {
+        result.push(currentText.slice(leftoverIndex).trim());
+      }
+    }
+    return result;
+  }
+
+  public async handleNewText(currentText: string | undefined, isLoading: boolean) {
+    if (!this.lastLoading && isLoading) {
+      this.reset();
+    }
+    this.lastLoading = isLoading;
+
+    if (!currentText) {
+      return;
+    }
+
+    const sentences = this.getCompletedSentences(currentText, isLoading);
+
+    for (let i = 0; i < sentences.length; i++) {
+      const sentence = sentences[i];
+      if (!this.fetchedSentences.has(sentence)) {
+        this.fetchedSentences.add(sentence);
+        const audioData = await this.generateSpeech(sentence);
+        await this.player.addChunk(audioData, i);
+      }
+    }
+  }
+
+  private async generateSpeech(sentence: string): Promise<ArrayBuffer> {
+    const blob = await this.voiceBackend(sentence, this.voice, 1.0);
+    return await blob.arrayBuffer();
+  }
+
+  public play() {
+    this.player.playAgain();
+  }
+
+  public stop() {
+    this.player.stopPlayback();
+  }
+
+  private reset() {
+    this.stop();
+    this.fetchedSentences.clear();
+    this.player.reset();
+  }
+
+  public setVolume(volume: number) {
+    this.player.setVolume(volume);
+  }
+
+  public setOnLoudnessChange(callback: (value: number) => void) {
+    this.player.setOnLoudnessChange((loudness) => {
+      callback(loudness);
+    });
+  }
+
+  public setOnEndOfSpeech(callback: () => void) {
+    this.player.setOnEndOfSpeech(callback);
+  }
+}