@rimori/client 1.0.0

package/src/components/Spinner.tsx

@@ -0,0 +1,24 @@
+import React from 'react';
+
+interface SpinnerProps {
+    text?: string;
+    size?: string
+    className?: string;
+}
+
+const Spinner: React.FC<SpinnerProps> = ({ text, className, size = "30px" }) => {
+    return (
+        <div className={"flex items-center space-x-2 " + className}>
+            <svg
+                style={{ width: size, height: size }}
+                className="animate-spin -ml-1 mr-3 h-5 w-5 text-white"
+                xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
+                <circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
+                <path className="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
+            </svg>
+            {text && <span className="">{text}</span>}
+        </div>
+    );
+};
+
+export default Spinner;

package/src/components/audio/Playbutton.tsx

@@ -0,0 +1,119 @@
+import React, { useState, useEffect } from 'react';
+import { FaPlayCircle, FaStopCircle } from "react-icons/fa";
+import { usePlugin } from "../../providers/PluginProvider";
+import Spinner from '../Spinner';
+import { EmitterSingleton } from '../../providers/EventEmitter';
+
+type AudioPlayerProps = {
+    text: string;
+    voice?: string;
+    language?: string;
+    hide?: boolean;
+    playOnMount?: boolean;
+    initialSpeed?: number;
+    enableSpeedAdjustment?: boolean;
+    playListenerEvent?: string;
+};
+
+export const AudioPlayOptions = [0.8, 0.9, 1.0, 1.1, 1.2, 1.5];
+export type AudioPlayOptionType = 0.8 | 0.9 | 1.0 | 1.1 | 1.2 | 1.5;
+
+let isFetchingAudio = false;
+
+export const AudioPlayer: React.FC<AudioPlayerProps> = ({
+    text,
+    voice,
+    language,
+    hide,
+    playListenerEvent,
+    initialSpeed = 1.0,
+    playOnMount = false,
+    enableSpeedAdjustment = false,
+}) => {
+    const [audioUrl, setAudioUrl] = useState<string | null>(null);
+    const [speed, setSpeed] = useState(initialSpeed);
+    const [isPlaying, setIsPlaying] = useState(false);
+    const [isLoading, setIsLoading] = useState(false);
+    const { getVoiceResponse, } = usePlugin();
+    const emitter = EmitterSingleton;
+
+    useEffect(() => {
+        if (!playListenerEvent) return;
+        emitter.on(playListenerEvent, () => togglePlayback());
+    }, [playListenerEvent]);
+
+    useEffect(() => {
+        audioUrl && setAudioUrl(null);
+        return () => {
+            audioUrl && URL.revokeObjectURL(audioUrl);
+        }
+    }, [text]);
+
+    // Function to generate audio from text using API
+    const generateAudio = async () => {
+        setIsLoading(true);
+
+        const blob = await getVoiceResponse(text, voice || (language ? "aws_default" : "openai_alloy"), 1, language);
+        setAudioUrl(URL.createObjectURL(blob));
+        setIsLoading(false);
+    };
+
+    // Effect to play audio when audioUrl changes and play state is true
+    useEffect(() => {
+        if (!audioUrl || !isPlaying) return;
+        const audio = new Audio(audioUrl);
+        audio.playbackRate = speed;
+        audio.play().then(() => {
+            audio.onended = () => {
+                setIsPlaying(false);
+                isFetchingAudio = false;
+            };
+        }).catch(e => {
+            console.warn("Error playing audio:", e);
+            setIsPlaying(false);
+        });
+
+        return () => {
+            audio.pause();
+        };
+    }, [audioUrl, isPlaying, speed]);
+
+    const togglePlayback = () => {
+        if (!isPlaying && !audioUrl) {
+            generateAudio().then(() => setIsPlaying(true));
+        } else {
+            setIsPlaying((prev) => !prev);
+        }
+    };
+
+    useEffect(() => {
+        if (!playOnMount || isFetchingAudio) return;
+        isFetchingAudio = true;
+        // console.log("playOnMount", playOnMount);
+        togglePlayback();
+    }, [playOnMount]);
+
+    return (
+        <div className="group relative">
+            <div className='flex flex-row items-end'>
+                {!hide && <button className="text-gray-500" onClick={togglePlayback} disabled={isLoading}>
+                    {isLoading ? <Spinner /> : isPlaying ? <FaStopCircle size={"25px"} /> : <FaPlayCircle size={"25px"} />}
+                </button>}
+                {enableSpeedAdjustment && (
+                    <div className="ml-1 opacity-0 group-hover:opacity-100 transition-opacity duration-300 flex flex-row text-sm text-gray-500">
+                        <span className='pr-1'>Speed: </span>
+                        <select
+                            value={speed}
+                            className='appearance-none cursor-pointer pr-0 p-0 rounded shadow leading-tight focus:outline-none focus:bg-gray-800 focus:ring bg-transparent border-0'
+                            onChange={(e) => setSpeed(parseFloat(e.target.value))}
+                            disabled={isLoading}>
+                            {AudioPlayOptions.map((s) => (
+                                <option key={s} value={s}>{s}</option>
+                            ))}
+                        </select>
+                    </div>
+                )}
+            </div>
+        </div>
+    );
+};

package/src/controller/AIController.ts

@@ -0,0 +1,87 @@
+import { env } from "../utils/constants";
+
+export interface ToolInvocation {
+    toolName: string;
+    args: Record<string, string>;
+}
+
+export interface Tool {
+    name: string;
+    description: string;
+    parameters: {
+        name: string;
+        type: "string" | "number" | "boolean";
+        description: string;
+    }[];
+}
+
+export interface Message {
+    id: string;
+    role: string;
+    content: string;
+    toolInvocations?: ToolInvocation[];
+}
+
+export async function generateText(messages: Message[], tools: Tool[], token: string) {
+    const response = await fetch(`${env.SUPABASE_URL}/functions/v1/llm`, {
+        method: 'POST',
+        body: JSON.stringify({ messages, tools }),
+        headers: { 'Authorization': `Bearer ${token}` }
+    });
+
+    return await response.json();
+}
+
+export type OnLLMResponse = (id: string, response: string, finished: boolean, toolInvocations?: ToolInvocation[]) => void;
+
+export async function streamChatGPT(messages: Message[], tools: Tool[], onResponse: OnLLMResponse, token: string) {
+    const messageId = Math.random().toString(36).substring(3);
+    const response = await fetch(`${env.SUPABASE_URL}/functions/v1/llm`, {
+        method: 'POST',
+        body: JSON.stringify({ messages, tools, stream: true }),
+        headers: { 'Authorization': `Bearer ${token}` }
+    });
+
+    if (!response.body) {
+        console.error('No response body.');
+        return;
+    }
+
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder('utf-8');
+
+    let content = "";
+    let done = false;
+    let toolInvocations: ToolInvocation[] = [];
+    while (!done) {
+        const { value } = await reader.read();
+
+        if (value) {
+            const chunk = decoder.decode(value, { stream: true });
+            const lines = chunk.split('\n').filter(line => line.trim() !== '');
+
+            for (const line of lines) {
+                const data = line.substring(3, line.length - 1);
+                const command = line.substring(0, 1);
+                // console.log("data: ", { line, data, command });
+
+                if (command === '0') {
+                    content += data;
+                    // console.log("AI response:", content);
+
+                    //content \n\n should be real line break when message is displayed
+                    onResponse(messageId, content.replace(/\\n/g, '\n'), false);
+                } else if (command === 'd') {
+                    // console.log("AI usage:", JSON.parse(line.substring(2)));
+                    done = true;
+                    break;
+                } else if (command === '9') {
+                    // console.log("tool call:", JSON.parse(line.substring(2)));
+                    // console.log("tools", tools);
+                    toolInvocations.push(JSON.parse(line.substring(2)));
+                }
+            }
+        }
+    }
+    onResponse(messageId, content.replace(/\\n/g, '\n'), true, toolInvocations);
+}

package/src/controller/ObjectController.ts

@@ -0,0 +1,109 @@
+import { env } from "../utils/constants";
+
+type PrimitiveType = 'string' | 'number' | 'boolean';
+
+// This is the type that can appear in the `type` property
+type ObjectToolParameterType =
+    | PrimitiveType
+    | { [key: string]: ObjectToolParameter }  // for nested objects
+    | [{ [key: string]: ObjectToolParameter }];  // for arrays of objects (notice the tuple type)
+
+interface ObjectToolParameter {
+    type: ObjectToolParameterType;
+    description?: string;
+    enum?: string[];
+}
+
+export type ObjectTool = {
+    [key: string]: ObjectToolParameter;
+};
+
+export interface ObjectRequest {
+    /**
+     * The tools that the AI can use.
+     */
+    tool: ObjectTool;
+    /**
+     * High level instructions for the AI to follow. Behaviour, tone, restrictions, etc.
+     * Example: "Act like a recipe writer."
+     */
+    behaviour?: string;
+    /**
+     * The specific instruction for the AI to follow.
+     * Example: "Generate a recipe using chicken, rice and vegetables."
+     */
+    instructions: string;
+}
+
+export async function generateObject(request: ObjectRequest, token: string) {
+    return await fetch(`${env.SUPABASE_URL}/functions/v1/llm-object`, {
+        method: 'POST',
+        body: JSON.stringify({
+            stream: false,
+            tool: request.tool,
+            behaviour: request.behaviour,
+            instructions: request.instructions,
+        }),
+        headers: { 'Authorization': `Bearer ${token}` }
+    }).then(response => response.json());
+}
+
+// TODO adjust stream to work with object
+export type OnLLMResponse = (id: string, response: string, finished: boolean, toolInvocations?: any[]) => void;
+
+export async function streamObject(request: ObjectRequest, onResponse: OnLLMResponse, token: string) {
+    const messageId = Math.random().toString(36).substring(3);
+    const response = await fetch(`${env.SUPABASE_URL}/functions/v1/llm-object`, {
+        method: 'POST',
+        body: JSON.stringify({
+            stream: true,
+            tools: request.tool,
+            systemInstructions: request.behaviour,
+            secondaryInstructions: request.instructions,
+        }),
+        headers: { 'Authorization': `Bearer ${token}` }
+    });
+
+    if (!response.body) {
+        console.error('No response body.');
+        return;
+    }
+
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder('utf-8');
+
+    let content = "";
+    let done = false;
+    let toolInvocations: any[] = [];
+    while (!done) {
+        const { value } = await reader.read();
+
+        if (value) {
+            const chunk = decoder.decode(value, { stream: true });
+            const lines = chunk.split('\n').filter(line => line.trim() !== '');
+
+            for (const line of lines) {
+                const data = line.substring(3, line.length - 1);
+                const command = line.substring(0, 1);
+                // console.log("data: ", { line, data, command });
+
+                if (command === '0') {
+                    content += data;
+                    // console.log("AI response:", content);
+
+                    //content \n\n should be real line break when message is displayed
+                    onResponse(messageId, content.replace(/\\n/g, '\n'), false);
+                } else if (command === 'd') {
+                    // console.log("AI usage:", JSON.parse(line.substring(2)));
+                    done = true;
+                    break;
+                } else if (command === '9') {
+                    // console.log("tool call:", JSON.parse(line.substring(2)));
+                    // console.log("tools", tools);
+                    toolInvocations.push(JSON.parse(line.substring(2)));
+                }
+            }
+        }
+    }
+    onResponse(messageId, content.replace(/\\n/g, '\n'), true, toolInvocations);
+}

package/src/controller/SettingsController.ts

@@ -0,0 +1,87 @@
+import { SupabaseClient } from "@supabase/supabase-js";
+import { LanguageLevel } from "../utils/difficultyConverter";
+
+type SettingsType = "user" | "system" | "plugin";
+
+export interface UserSettings {
+    motherTongue: string;
+    languageLevel: LanguageLevel;
+}
+
+export interface SystemSettings {
+    // TODO: add system settings
+}
+
+export class SettingsController {
+    private pluginId: string;
+    private supabase: SupabaseClient;
+
+    constructor(supabase: SupabaseClient, pluginId: string) {
+        this.supabase = supabase;
+        this.pluginId = pluginId;
+    }
+
+    private getSettingsType(genericSettings?: "user" | "system"): SettingsType {
+        return genericSettings || "plugin";
+    }
+
+    private async fetchSettings(type: SettingsType): Promise<any | null> {
+        const pluginId = type === "plugin" ? this.pluginId : type;
+        const { data } = await this.supabase.from("plugin_settings").select("*").eq("plugin_id", pluginId)
+
+        if (!data || data.length === 0) {
+            return null;
+        }
+
+        return data[0].settings;
+    }
+
+    private async saveSettings(settings: any, type: SettingsType): Promise<void> {
+        if (type !== "plugin") {
+            throw new Error(`Cannot modify ${type} settings`);
+        }
+
+        await this.supabase.from("plugin_settings").upsert({ plugin_id: this.pluginId, settings });
+    }
+
+    /**
+     * Get the settings for the plugin. T can be any type of settings, UserSettings or SystemSettings.
+     * @param defaultSettings The default settings to use if no settings are found.
+     * @param genericSettings The type of settings to get.
+     * @returns The settings for the plugin. 
+     */
+    public async getSettings<T extends object>(defaultSettings: T, genericSettings?: "user" | "system"): Promise<T> {
+        const type = this.getSettingsType(genericSettings);
+        const storedSettings = await this.fetchSettings(type) as T | null;
+
+        if (!storedSettings) {
+            if (type === "plugin") {
+                await this.saveSettings(defaultSettings, type);
+            }
+            return defaultSettings;
+        }
+
+        // Handle settings migration
+        const storedKeys = Object.keys(storedSettings);
+        const defaultKeys = Object.keys(defaultSettings);
+
+        if (storedKeys.length !== defaultKeys.length) {
+            const validStoredSettings = Object.fromEntries(
+                Object.entries(storedSettings).filter(([key]) => defaultKeys.includes(key))
+            );
+            const mergedSettings = { ...defaultSettings, ...validStoredSettings } as T;
+
+            if (type === "plugin") {
+                await this.saveSettings(mergedSettings, type);
+            }
+            return mergedSettings;
+        }
+
+        return storedSettings;
+    }
+
+    public async setSettings(settings: any, genericSettings?: "user" | "system"): Promise<void> {
+        const type = this.getSettingsType(genericSettings);
+        await this.saveSettings(settings, type);
+    }
+}

package/src/controller/SharedContentController.ts

@@ -0,0 +1,71 @@
+import { v4 as uuidv4 } from 'uuid';
+import { ObjectRequest } from "./ObjectController";
+import { RimoriClient } from "../plugin/RimoriClient";
+
+export interface BasicAssignment {
+    id: string;
+    createdAt: Date;
+    topic: string;
+    createdBy: string;
+    verified: boolean;
+    keywords: any;
+}
+
+export class SharedContentController {
+    private rimoriClient: RimoriClient;
+
+    constructor(rimoriClient: RimoriClient) {
+        this.rimoriClient = rimoriClient;
+    }
+
+    public async fetchNewSharedContent<T, R = T & BasicAssignment>(
+        type: string,
+        generatorInstructions: (reservedTopics: string[]) => Promise<ObjectRequest> | ObjectRequest,
+        filter?: { column: string, value: string | number | boolean },
+    ): Promise<R[]> {
+        const queryParameter = { filter_column: filter?.column || null, filter_value: filter?.value || null, unread: true }
+        const { data: newAssignments } = await this.rimoriClient.rpc(type + "_entries", queryParameter)
+        console.log('newAssignments:', newAssignments);
+
+        if ((newAssignments as any[]).length > 0) {
+            return newAssignments as R[];
+        }
+        // generate new assignments
+        const { data: oldAssignments } = await this.rimoriClient.rpc(type + "_entries", { ...queryParameter, unread: false })
+        console.log('oldAssignments:', oldAssignments);
+        const reservedTopics = this.getReservedTopics(oldAssignments as BasicAssignment[]);
+
+        const request = await generatorInstructions(reservedTopics);
+        if (!request.tool.keywords || !request.tool.topic) {
+            throw new Error("topic or keywords not found in the request schema");
+        }
+        const instructions = await this.rimoriClient.generateObject(request);
+        console.log('instructions:', instructions);
+
+        const preparedData = {
+            id: uuidv4(),
+            ...instructions,
+            keywords: this.purifyStringArray(instructions.keywords),
+        };
+        return await this.rimoriClient.from(type).insert(preparedData).then(() => [preparedData] as R[]);
+    }
+
+    private getReservedTopics(oldAssignments: BasicAssignment[]) {
+        return oldAssignments.map(({ topic, keywords }) => {
+            const keywordTexts = this.purifyStringArray(keywords).join(',');
+            return `${topic}(${keywordTexts})`;
+        });
+    }
+
+    private purifyStringArray(array: { text: string }[]): string[] {
+        return array.map(({ text }) => text);
+    }
+
+    public async getSharedContent<T extends BasicAssignment>(type: string, id: string): Promise<T> {
+        return await this.rimoriClient.from(type).select().eq('id', id).single() as unknown as T;
+    }
+
+    public async completeSharedContent(type: string, assignmentId: string) {
+        await this.rimoriClient.from(type + "_result").insert({ assignment_id: assignmentId });
+    }
+}

package/src/controller/VoiceController.ts

@@ -0,0 +1,26 @@
+import { SupabaseClient } from "@supabase/supabase-js";
+
+export async function getSTTResponse(supabase: SupabaseClient, audio: Blob) {
+    const formData = new FormData();
+    formData.append('file', audio);
+
+    return await supabase.functions.invoke('speech', { method: 'POST', body: formData }).then((r: any) => r.text);
+}
+
+export async function getTTSResponse(supabaseUrl: string, request: TTSRequest, token: string) {
+    return await fetch(`${supabaseUrl}/functions/v1/speech`, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${token}`
+        },
+        body: JSON.stringify(request),
+    }).then(r => r.blob());
+}
+
+interface TTSRequest {
+    input: string;
+    voice: string;
+    speed: number;
+    language?: string;
+}

package/src/hooks/UseChatHook.ts

@@ -0,0 +1,25 @@
+import React from "react";
+import { usePlugin } from "../providers/PluginProvider";
+import { ToolInvocation, Tool, Message } from "../controller/AIController";
+
+export function useChat(tools?: Tool[]) {
+    const [messages, setMessages] = React.useState<Message[]>([]);
+    const [isLoading, setIsLoading] = React.useState(false);
+    const { getAIResponseStream } = usePlugin();
+
+    const append = (appendMessages: Message[]) => {
+        getAIResponseStream([...messages, ...appendMessages], (id, message, finished: boolean, toolInvocations?: ToolInvocation[]) => {
+            const lastMessage = messages[messages.length - 1];
+            setIsLoading(!finished);
+
+            if (lastMessage?.id === id) {
+                lastMessage.content = message;
+                setMessages([...messages, lastMessage]);
+            } else {
+                setMessages([...messages, ...appendMessages, { id, role: 'assistant', content: message, toolInvocations }]);
+            }
+        }, tools);
+    };
+
+    return { messages, append, isLoading, setMessages, lastMessage: messages[messages.length - 1] as Message | undefined };
+}

package/src/index.ts

@@ -0,0 +1,14 @@
+export * from "./components/MarkdownEditor";
+export * from "./components/CRUDModal";
+export * from "./plugin/ThemeSetter";
+export * from "./components/audio/Playbutton";
+export * from "./providers/EventEmitterContext";
+export * from "./providers/PluginProvider";
+export * from "./providers/EventEmitter";
+export * from "./components/Spinner";
+export * from "./hooks/UseChatHook";
+export * from "./utils/PluginUtils";
+export * from "./plugin/RimoriClient";
+export * from "./controller/AIController";
+export * from "./controller/SharedContentController";
+export * from "./controller/SettingsController";