@rimori/react-client 0.1.0

package/src/components/ai/EmbeddedAssistent/TTS/Player.ts

@@ -0,0 +1,197 @@
+export class ChunkedAudioPlayer {
+  private audioContext!: AudioContext;
+  private chunkQueue: ArrayBuffer[] = [];
+  private isPlaying = false;
+  private analyser!: AnalyserNode;
+  private dataArray!: Uint8Array<ArrayBuffer>;
+  private shouldMonitorLoudness = true;
+  private isMonitoring = false;
+  private handle = 0;
+  private volume = 1.0;
+  private loudnessCallback: (value: number) => void = () => {};
+  private currentIndex = 0;
+  private startedPlaying = false;
+  private onEndOfSpeech: () => void = () => {};
+
+  constructor() {
+    this.init();
+  }
+
+  private init(): void {
+    this.audioContext = new AudioContext();
+    this.analyser = this.audioContext.createAnalyser();
+    this.analyser.fftSize = 256; // Set the FFT size (smaller values provide faster updates, larger ones give better resolution)
+    const bufferLength = this.analyser.frequencyBinCount;
+    this.dataArray = new Uint8Array(bufferLength); // Array to hold frequency data
+  }
+
+  public setOnLoudnessChange(callback: (value: number) => void) {
+    this.loudnessCallback = callback;
+  }
+
+  public setVolume(volume: number) {
+    this.volume = volume;
+  }
+
+  public async addChunk(chunk: ArrayBuffer, position: number): Promise<void> {
+    console.log('Adding chunk', position, chunk);
+    this.chunkQueue[position] = chunk;
+    // console.log("received chunk", {
+    //     chunkQueue: this.chunkQueue.length,
+    //     isPlaying: this.isPlaying,
+    // })
+
+    if (position === 0 && !this.startedPlaying) {
+      this.startedPlaying = true;
+      this.playChunks();
+    }
+  }
+
+  private playChunks(): void {
+    // console.log({ isPlaying: this.isPlaying });
+    if (this.isPlaying) return;
+    if (!this.chunkQueue[this.currentIndex]) {
+      // wait until the correct chunk arrives
+      setTimeout(() => this.playChunks(), 10);
+    }
+    this.isPlaying = true;
+
+    this.playChunk(this.chunkQueue[this.currentIndex]).then(() => {
+      this.isPlaying = false;
+      this.currentIndex++;
+      if (this.chunkQueue[this.currentIndex]) {
+        this.shouldMonitorLoudness = true;
+        this.playChunks();
+      } else {
+        // console.log('Playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
+        setTimeout(() => {
+          // console.log('Check again if really playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
+          if (this.chunkQueue.length > this.currentIndex) {
+            this.playChunks();
+          } else {
+            this.startedPlaying = false;
+            this.shouldMonitorLoudness = false;
+          }
+        }, 1000);
+      }
+    });
+  }
+
+  public stopPlayback(): void {
+    // console.log('Stopping playback');
+    // Implement logic to stop the current playback
+    this.isPlaying = false;
+    this.chunkQueue = [];
+    this.startedPlaying = false;
+    this.shouldMonitorLoudness = false;
+    cancelAnimationFrame(this.handle);
+  }
+
+  private playChunk(chunk: ArrayBuffer): Promise<void> {
+    // console.log({queue: this.chunkQueue})
+    if (!chunk) {
+      return Promise.resolve();
+    }
+
+    // console.log('Playing chunk', chunk);
+    return new Promise((resolve) => {
+      const source = this.audioContext.createBufferSource();
+      this.audioContext.decodeAudioData(chunk.slice(0)).then((audioBuffer) => {
+        source.buffer = audioBuffer;
+
+        // Create a GainNode for volume control
+        const gainNode = this.audioContext.createGain();
+        gainNode.gain.value = this.volume;
+
+        // Connect the source to the GainNode, then to the analyser node, then to the destination (speakers)
+        source.connect(gainNode);
+        gainNode.connect(this.analyser);
+        this.analyser.connect(this.audioContext.destination);
+
+        source.start(0);
+        // console.log('Playing chunk', this.currentIndex);
+        gainNode.gain.value = this.volume;
+        source.onended = () => {
+          // console.log('Chunk ended');
+          resolve();
+        };
+
+        // Start monitoring loudness only once
+        if (!this.isMonitoring) {
+          this.isMonitoring = true;
+          this.shouldMonitorLoudness = true;
+          this.monitorLoudness();
+        }
+      });
+    });
+  }
+
+  async playAgain(): Promise<void> {
+    console.log('Playing again');
+    if (this.chunkQueue.length > 0 && !this.isPlaying) {
+      this.playChunks();
+    }
+  }
+
+  private monitorLoudness(): void {
+    // Stop monitoring when the flag is false
+    if (!this.shouldMonitorLoudness) {
+      // console.log('Loudness monitoring stopped.');
+      cancelAnimationFrame(this.handle);
+      this.loudnessCallback(0);
+      this.onEndOfSpeech();
+      return;
+    }
+
+    // Get the time domain data from the analyser (this is a snapshot of the waveform)
+    this.analyser.getByteTimeDomainData(this.dataArray);
+
+    // Calculate the RMS (root mean square) of the waveform values to get the perceived loudness
+    let sum = 0;
+    for (let i = 0; i < this.dataArray.length; i++) {
+      const value = this.dataArray[i] / 128.0 - 1.0; // Normalize between -1 and 1
+      sum += value * value;
+    }
+
+    const rms = Math.sqrt(sum / this.dataArray.length);
+
+    // Handle the case where RMS is 0 to avoid log10(0)
+    if (rms === 0) {
+      // console.log('Current loudness: Silent');
+    } else {
+      let loudnessInDb = 20 * Math.log10(rms); // Convert to dB
+      // console.log('Current loudness:' + loudnessInDb);
+      const minDb = -57;
+      const maxDb = -15;
+
+      if (loudnessInDb < minDb) {
+        loudnessInDb = minDb;
+      }
+      if (loudnessInDb > maxDb) {
+        loudnessInDb = maxDb;
+      }
+
+      const loudnessScale = ((loudnessInDb - minDb) / (maxDb - minDb)) * 100;
+      // console.log("root:corrent loudness", loudnessScale);
+
+      this.loudnessCallback(loudnessScale);
+    }
+
+    // Call this method again at regular intervals if you want continuous loudness monitoring
+    this.handle = requestAnimationFrame(() => this.monitorLoudness());
+  }
+  public reset() {
+    // console.log('Resetting player');
+    this.stopPlayback();
+    this.currentIndex = 0;
+    this.shouldMonitorLoudness = true;
+    //reset to the beginning when the class gets initialized
+    this.isMonitoring = false;
+    this.isPlaying = false;
+    this.init();
+  }
+
+  public setOnEndOfSpeech(callback: () => void) {
+    this.onEndOfSpeech = callback;
+  }
+}

package/src/components/ai/EmbeddedAssistent/VoiceRecoder.tsx

@@ -0,0 +1,129 @@
+import { useRimori } from '../../../providers/PluginProvider';
+import { FaMicrophone, FaSpinner } from 'react-icons/fa6';
+import { AudioController } from '@rimori/client';
+import { useState, useRef, forwardRef, useImperativeHandle, useEffect } from 'react';
+
+interface Props {
+  iconSize?: string;
+  className?: string;
+  disabled?: boolean;
+  loading?: boolean;
+  enablePushToTalk?: boolean;
+  onRecordingStatusChange: (running: boolean) => void;
+  onVoiceRecorded: (message: string) => void;
+}
+
+export const VoiceRecorder = forwardRef(
+  (
+    {
+      onVoiceRecorded,
+      iconSize,
+      className,
+      disabled,
+      loading,
+      onRecordingStatusChange,
+      enablePushToTalk = false,
+    }: Props,
+    ref,
+  ) => {
+    const [isRecording, setIsRecording] = useState(false);
+    const [internalIsProcessing, setInternalIsProcessing] = useState(false);
+    const audioControllerRef = useRef<AudioController | null>(null);
+    const { ai, plugin } = useRimori();
+
+    // Ref for latest onVoiceRecorded callback
+    const onVoiceRecordedRef = useRef(onVoiceRecorded);
+    useEffect(() => {
+      onVoiceRecordedRef.current = onVoiceRecorded;
+    }, [onVoiceRecorded]);
+
+    const startRecording = async () => {
+      try {
+        if (!audioControllerRef.current) {
+          audioControllerRef.current = new AudioController(plugin.pluginId);
+        }
+
+        await audioControllerRef.current.startRecording();
+        setIsRecording(true);
+        onRecordingStatusChange(true);
+      } catch (error) {
+        console.error('Failed to start recording:', error);
+        // Handle permission denied or other errors
+      }
+    };
+
+    const stopRecording = async () => {
+      try {
+        if (audioControllerRef.current && isRecording) {
+          const audioResult = await audioControllerRef.current.stopRecording();
+          // console.log("audioResult: ", audioResult);
+
+          setInternalIsProcessing(true);
+
+          // Play the recorded audio from the Blob
+          // const blobUrl = URL.createObjectURL(audioResult.recording);
+          // const audioRef = new Audio(blobUrl);
+          // audioRef.onended = () => URL.revokeObjectURL(blobUrl);
+          // audioRef.play().catch((e) => console.error('Playback error:', e));
+
+          // console.log("audioBlob: ", audioResult.recording);
+          const text = await ai.getTextFromVoice(audioResult.recording);
+          // console.log("stt result", text);
+          // throw new Error("test");
+          setInternalIsProcessing(false);
+          onVoiceRecordedRef.current(text);
+        }
+      } catch (error) {
+        console.error('Failed to stop recording:', error);
+      } finally {
+        setIsRecording(false);
+        onRecordingStatusChange(false);
+      }
+    };
+
+    useImperativeHandle(ref, () => ({
+      startRecording,
+      stopRecording,
+    }));
+
+    // push to talk feature
+    const spacePressedRef = useRef(false);
+
+    useEffect(() => {
+      if (!enablePushToTalk) return;
+
+      const handleKeyDown = async (event: KeyboardEvent) => {
+        if (event.code === 'Space' && !spacePressedRef.current) {
+          spacePressedRef.current = true;
+          await startRecording();
+        }
+      };
+      const handleKeyUp = (event: KeyboardEvent) => {
+        if (event.code === 'Space' && spacePressedRef.current) {
+          spacePressedRef.current = false;
+          stopRecording();
+        }
+      };
+      window.addEventListener('keydown', handleKeyDown);
+      window.addEventListener('keyup', handleKeyUp);
+      return () => {
+        window.removeEventListener('keydown', handleKeyDown);
+        window.removeEventListener('keyup', handleKeyUp);
+      };
+    }, [enablePushToTalk]);
+
+    return (
+      <button
+        className={'flex flex-row justify-center items-center rounded-full mx-auto disabled:opacity-50 ' + className}
+        onClick={isRecording ? stopRecording : startRecording}
+        disabled={disabled || loading || internalIsProcessing}
+      >
+        {loading || internalIsProcessing ? (
+          <FaSpinner className="animate-spin" />
+        ) : (
+          <FaMicrophone size={iconSize} className={isRecording ? 'text-red-600' : ''} />
+        )}
+      </button>
+    );
+  },
+);

package/src/components/ai/utils.ts

@@ -0,0 +1,21 @@
+export interface FirstMessages {
+  instructions?: string;
+  userMessage?: string;
+  assistantMessage?: string;
+}
+
+export function getFirstMessages(instructions: FirstMessages): any[] {
+  const messages = [];
+
+  if (instructions.instructions) {
+    messages.push({ id: '1', role: 'system', content: instructions.instructions });
+  }
+  if (instructions.userMessage) {
+    messages.push({ id: '2', role: 'user', content: instructions.userMessage });
+  }
+  if (instructions.assistantMessage) {
+    messages.push({ id: '3', role: 'assistant', content: instructions.assistantMessage });
+  }
+
+  return messages;
+}

package/src/components/audio/Playbutton.tsx

@@ -0,0 +1,126 @@
+import React, { useState, useEffect } from 'react';
+import { FaPlayCircle, FaStopCircle } from 'react-icons/fa';
+import { useRimori } from '../../providers/PluginProvider';
+import { Spinner } from '../Spinner';
+import { EventBus } from '@rimori/client';
+
+type AudioPlayerProps = {
+  text: string;
+  voice?: string;
+  language?: string;
+  hide?: boolean;
+  playOnMount?: boolean;
+  initialSpeed?: number;
+  enableSpeedAdjustment?: boolean;
+  playListenerEvent?: string;
+};
+
+export const AudioPlayOptions = [0.8, 0.9, 1.0, 1.1, 1.2, 1.5];
+export type AudioPlayOptionType = 0.8 | 0.9 | 1.0 | 1.1 | 1.2 | 1.5;
+
+let isFetchingAudio = false;
+
+export const AudioPlayer: React.FC<AudioPlayerProps> = ({
+  text,
+  voice,
+  language,
+  hide,
+  playListenerEvent,
+  initialSpeed = 1.0,
+  playOnMount = false,
+  enableSpeedAdjustment = false,
+}) => {
+  const [audioUrl, setAudioUrl] = useState<string | null>(null);
+  const [speed, setSpeed] = useState(initialSpeed);
+  const [isPlaying, setIsPlaying] = useState(false);
+  const [isLoading, setIsLoading] = useState(false);
+  const { ai } = useRimori();
+
+  useEffect(() => {
+    if (audioUrl) setAudioUrl(null);
+    return () => {
+      if (audioUrl) URL.revokeObjectURL(audioUrl);
+    };
+  }, [text]);
+
+  // Function to generate audio from text using API
+  const generateAudio = async () => {
+    setIsLoading(true);
+
+    const blob = await ai.getVoice(text, voice || (language ? 'aws_default' : 'openai_alloy'), 1, language);
+    setAudioUrl(URL.createObjectURL(blob));
+    setIsLoading(false);
+  };
+
+  // Effect to play audio when audioUrl changes and play state is true
+  useEffect(() => {
+    if (!audioUrl || !isPlaying) return;
+    const audio = new Audio(audioUrl);
+    audio.playbackRate = speed;
+    audio
+      .play()
+      .then(() => {
+        audio.onended = () => {
+          setIsPlaying(false);
+          isFetchingAudio = false;
+        };
+      })
+      .catch((e) => {
+        console.warn('Error playing audio:', e);
+        setIsPlaying(false);
+      });
+
+    return () => {
+      audio.pause();
+    };
+  }, [audioUrl, isPlaying, speed]);
+
+  const togglePlayback = () => {
+    if (!isPlaying && !audioUrl) {
+      generateAudio().then(() => setIsPlaying(true));
+    } else {
+      setIsPlaying((prev) => !prev);
+    }
+  };
+
+  useEffect(() => {
+    if (!playListenerEvent) return;
+    EventBus.on(playListenerEvent, () => togglePlayback());
+  }, [playListenerEvent]);
+
+  useEffect(() => {
+    if (!playOnMount || isFetchingAudio) return;
+    isFetchingAudio = true;
+    // console.log("playOnMount", playOnMount);
+    togglePlayback();
+  }, [playOnMount]);
+
+  return (
+    <div className="group relative">
+      <div className="flex flex-row items-end">
+        {!hide && (
+          <button className="text-gray-500" onClick={togglePlayback} disabled={isLoading}>
+            {isLoading ? <Spinner /> : isPlaying ? <FaStopCircle size={'25px'} /> : <FaPlayCircle size={'25px'} />}
+          </button>
+        )}
+        {enableSpeedAdjustment && (
+          <div className="ml-1 opacity-0 group-hover:opacity-100 transition-opacity duration-300 flex flex-row text-sm text-gray-500">
+            <span className="pr-1">Speed: </span>
+            <select
+              value={speed}
+              className="appearance-none cursor-pointer pr-0 p-0 rounded shadow leading-tight focus:outline-none focus:bg-gray-800 focus:ring bg-transparent border-0"
+              onChange={(e) => setSpeed(parseFloat(e.target.value))}
+              disabled={isLoading}
+            >
+              {AudioPlayOptions.map((s) => (
+                <option key={s} value={s}>
+                  {s}
+                </option>
+              ))}
+            </select>
+          </div>
+        )}
+      </div>
+    </div>
+  );
+};

package/src/hooks/I18nHooks.ts

@@ -0,0 +1,33 @@
+import { TOptions } from '@rimori/client';
+import { useEffect, useState } from 'react';
+import { Translator } from '@rimori/client';
+import { useRimori } from '../providers/PluginProvider';
+
+type TranslatorFn = (key: string, options?: TOptions) => string;
+
+/**
+ * Custom useTranslation hook that provides a translation function and indicates readiness
+ * @returns An object containing the translation function (`t`) and a boolean (`ready`) indicating if the translator is initialized.
+ */
+export function useTranslation(): { t: TranslatorFn; ready: boolean } {
+  const { plugin } = useRimori();
+  const [translatorInstance, setTranslatorInstance] = useState<Translator | null>(null);
+
+  useEffect(() => {
+    void plugin.getTranslator().then(setTranslatorInstance);
+  }, [plugin]);
+
+  const safeT = (key: string, options?: TOptions): string => {
+    // return zero-width space if translator is not initialized to keep text space occupied
+    if (!translatorInstance) return '\u200B'; // zero-width space
+
+    const result = translatorInstance.t(key, options);
+    if (!result) {
+      console.error(`Translation key not found: ${key}`);
+      return '\u200B'; // zero-width space
+    }
+    return result;
+  };
+
+  return { t: safeT, ready: translatorInstance !== null };
+}

package/src/hooks/UseChatHook.ts

@@ -0,0 +1,38 @@
+import React from 'react';
+import { Tool } from '@rimori/client';
+import { useRimori } from '../providers/PluginProvider';
+import { Message, ToolInvocation } from '@rimori/client';
+
+export function useChat(tools?: Tool[]) {
+  const [messages, setMessages] = React.useState<Message[]>([]);
+  const [isLoading, setIsLoading] = React.useState(false);
+  const { ai } = useRimori();
+
+  const append = (appendMessages: Message[]) => {
+    const allMessages = [...messages, ...appendMessages];
+    setMessages(allMessages);
+    ai.getSteamedText(
+      allMessages,
+      (id, message, finished: boolean, toolInvocations?: ToolInvocation[]) => {
+        const lastMessage = messages[messages.length - 1];
+        setIsLoading(!finished);
+
+        if (lastMessage?.id === id) {
+          lastMessage.content = message;
+          setMessages([...messages, lastMessage]);
+        } else {
+          setMessages([...allMessages, { id, role: 'assistant', content: message, toolCalls: toolInvocations }]);
+        }
+      },
+      tools,
+    );
+  };
+
+  return {
+    messages,
+    append,
+    isLoading,
+    setMessages,
+    lastMessage: messages[messages.length - 1] as Message | undefined,
+  };
+}

package/src/plugin/ThemeSetter.ts

@@ -0,0 +1,23 @@
+export function setTheme(theme?: string | null) {
+  document.documentElement.classList.add('dark:text-gray-200');
+
+  if (isDarkTheme(theme)) {
+    document.documentElement.setAttribute('data-theme', 'dark');
+    document.documentElement.classList.add('dark', 'dark:bg-gray-950');
+    document.documentElement.style.background = 'hsl(var(--background))';
+  }
+}
+
+export function isDarkTheme(theme?: string | null): boolean {
+  // If no theme provided, try to get from URL as fallback (for standalone mode)
+  if (!theme) {
+    const urlParams = new URLSearchParams(window.location.search);
+    theme = urlParams.get('theme');
+  }
+
+  if (!theme || theme === 'system') {
+    return window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches;
+  }
+
+  return theme === 'dark';
+}