@rimori/client 1.1.10 → 1.2.0

package/dist/components/ai/EmbeddedAssistent/VoiceRecoder.d.ts

@@ -3,6 +3,7 @@ interface Props {
     className?: string;
     disabled?: boolean;
     loading?: boolean;
+    enablePushToTalk?: boolean;
     onRecordingStatusChange: (running: boolean) => void;
     onVoiceRecorded: (message: string) => void;
 }

package/dist/components/ai/EmbeddedAssistent/VoiceRecoder.js

@@ -11,12 +11,13 @@ import { jsx as _jsx } from "react/jsx-runtime";
 import { useState, useRef, forwardRef, useImperativeHandle, useEffect } from 'react';
 import { FaMicrophone, FaSpinner } from 'react-icons/fa6';
 import { usePlugin } from '../../../components';
-export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className, disabled, loading, onRecordingStatusChange }, ref) => {
+export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className, disabled, loading, onRecordingStatusChange, enablePushToTalk = false }, ref) => {
     const [isRecording, setIsRecording] = useState(false);
+    const [internalIsProcessing, setInternalIsProcessing] = useState(false);
     const mediaRecorderRef = useRef(null);
     const audioChunksRef = useRef([]);
     const mediaStreamRef = useRef(null);
-    const { llm } = usePlugin();
+    const { ai: llm } = usePlugin();
     // Ref for latest onVoiceRecorded callback
     const onVoiceRecordedRef = useRef(onVoiceRecorded);
     useEffect(() => {
@@ -33,7 +34,10 @@ export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className,
         mediaRecorder.onstop = () => __awaiter(void 0, void 0, void 0, function* () {
             const audioBlob = new Blob(audioChunksRef.current);
             audioChunksRef.current = [];
-            onVoiceRecordedRef.current(yield llm.getTextFromVoice(audioBlob));
+            setInternalIsProcessing(true);
+            const text = yield llm.getTextFromVoice(audioBlob);
+            setInternalIsProcessing(false);
+            onVoiceRecordedRef.current(text);
         });
         mediaRecorder.start();
         setIsRecording(true);
@@ -57,6 +61,8 @@ export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className,
     // push to talk feature
     const spacePressedRef = useRef(false);
     useEffect(() => {
+        if (!enablePushToTalk)
+            return;
         const handleKeyDown = (event) => __awaiter(void 0, void 0, void 0, function* () {
             if (event.code === 'Space' && !spacePressedRef.current) {
                 spacePressedRef.current = true;
@@ -75,7 +81,7 @@ export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className,
             window.removeEventListener('keydown', handleKeyDown);
             window.removeEventListener('keyup', handleKeyUp);
         };
-    }, []);
-    return (_jsx("button", { className: "w-16 h-16 flex text-4xl shadow-lg flex-row justify-center items-center rounded-full mx-auto bg-gray-400 dark:bg-gray-800 pl-[6px] disabled:opacity-50 " + className, onClick: isRecording ? stopRecording : startRecording, disabled: disabled || loading, children: loading ? _jsx(FaSpinner, { className: "animate-spin mr-[6px]" }) :
-            _jsx(FaMicrophone, { size: iconSize, className: "h-7 w-7 mr-2 " + (isRecording ? "text-red-600" : "") }) }));
+    }, [enablePushToTalk]);
+    return (_jsx("button", { className: "flex flex-row justify-center items-center rounded-full mx-auto disabled:opacity-50 " + className, onClick: isRecording ? stopRecording : startRecording, disabled: disabled || loading || internalIsProcessing, children: loading || internalIsProcessing ? _jsx(FaSpinner, { className: "animate-spin" }) :
+            _jsx(FaMicrophone, { size: iconSize, className: (isRecording ? "text-red-600" : "") }) }));
 });

package/dist/components/ai/utils.js

@@ -9,6 +9,5 @@ export function getFirstMessages(instructions) {
     if (instructions.assistantMessage) {
         messages.push({ id: '3', role: 'assistant', content: instructions.assistantMessage });
     }
-    console.log("getFirstMessages", messages);
     return messages;
 }

package/dist/components/audio/Playbutton.js

@@ -12,7 +12,7 @@ import { useState, useEffect } from 'react';
 import { FaPlayCircle, FaStopCircle } from "react-icons/fa";
 import { usePlugin } from "../../providers/PluginProvider";
 import { Spinner } from '../Spinner';
-import { EventBus } from '../../plugin/fromRimori/EventBus';
+import { EventBus } from '../../fromRimori/EventBus';
 export const AudioPlayOptions = [0.8, 0.9, 1.0, 1.1, 1.2, 1.5];
 let isFetchingAudio = false;
 export const AudioPlayer = ({ text, voice, language, hide, playListenerEvent, initialSpeed = 1.0, playOnMount = false, enableSpeedAdjustment = false, }) => {
@@ -20,7 +20,7 @@ export const AudioPlayer = ({ text, voice, language, hide, playListenerEvent, in
     const [speed, setSpeed] = useState(initialSpeed);
     const [isPlaying, setIsPlaying] = useState(false);
     const [isLoading, setIsLoading] = useState(false);
-    const { llm } = usePlugin();
+    const { ai } = usePlugin();
     useEffect(() => {
         if (!playListenerEvent)
             return;
@@ -35,7 +35,7 @@ export const AudioPlayer = ({ text, voice, language, hide, playListenerEvent, in
     // Function to generate audio from text using API
     const generateAudio = () => __awaiter(void 0, void 0, void 0, function* () {
         setIsLoading(true);
-        const blob = yield llm.getVoice(text, voice || (language ? "aws_default" : "openai_alloy"), 1, language);
+        const blob = yield ai.getVoice(text, voice || (language ? "aws_default" : "openai_alloy"), 1, language);
         setAudioUrl(URL.createObjectURL(blob));
         setIsLoading(false);
     });

package/dist/{core → components}/components/ContextMenu.js

@@ -1,6 +1,6 @@
 import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
 import { useState, useEffect, useRef } from "react";
-import { EventBus } from "../../plugin/fromRimori/EventBus";
+import { EventBus } from "../../fromRimori/EventBus";
 const ContextMenu = ({ client }) => {
     const [isOpen, setIsOpen] = useState(false);
     const [actions, setActions] = useState([]);
@@ -84,7 +84,7 @@ const ContextMenu = ({ client }) => {
                 var _a;
                 setIsOpen(false);
                 (_a = window.getSelection()) === null || _a === void 0 ? void 0 : _a.removeAllRanges();
-                client.event.emitSidebarAction(action.pluginId, action.actionKey, position.text);
+                client.event.emitSidebarAction(action.plugin_id, action.action_key, position.text);
             } }, index))) }));
 };
 function MenuEntryItem(props) {

package/dist/components.d.ts

@@ -1,10 +1,10 @@
-export * from "./components/MarkdownEditor";
+export * from "./components/ai/Assistant";
+export * from "./components/ai/Avatar";
+export * from "./components/ai/EmbeddedAssistent/VoiceRecoder";
+export * from "./components/audio/Playbutton";
 export * from "./components/CRUDModal";
+export * from "./components/MarkdownEditor";
 export * from "./components/Spinner";
-export * from "./components/audio/Playbutton";
 export * from "./hooks/UseChatHook";
 export * from "./plugin/ThemeSetter";
 export * from "./providers/PluginProvider";
-export * from "./components/ai/Avatar";
-export * from "./components/ai/Assistant";
-export * from "./types/Actions";

package/dist/components.js

@@ -1,11 +1,11 @@
 // React components and hooks exports
-export * from "./components/MarkdownEditor";
+export * from "./components/ai/Assistant";
+export * from "./components/ai/Avatar";
+export * from "./components/ai/EmbeddedAssistent/VoiceRecoder";
+export * from "./components/audio/Playbutton";
 export * from "./components/CRUDModal";
+export * from "./components/MarkdownEditor";
 export * from "./components/Spinner";
-export * from "./components/audio/Playbutton";
 export * from "./hooks/UseChatHook";
 export * from "./plugin/ThemeSetter";
 export * from "./providers/PluginProvider";
-export * from "./components/ai/Avatar";
-export * from "./components/ai/Assistant";
-export * from "./types/Actions";

package/dist/core/controller/AIController.d.ts

@@ -0,0 +1,15 @@
+import { Tool } from "../../fromRimori/PluginTypes";
+export interface ToolInvocation {
+    toolCallId: string;
+    toolName: string;
+    args: Record<string, string>;
+}
+export interface Message {
+    id?: string;
+    role: "user" | "assistant" | "system";
+    content: string;
+    toolCalls?: ToolInvocation[];
+}
+export declare function generateText(backendUrl: string, messages: Message[], tools: Tool[], token: string): Promise<any>;
+export type OnLLMResponse = (id: string, response: string, finished: boolean, toolInvocations?: ToolInvocation[]) => void;
+export declare function streamChatGPT(backendUrl: string, messages: Message[], tools: Tool[], onResponse: OnLLMResponse, token: string): Promise<void>;

package/dist/core/controller/AIController.js

@@ -0,0 +1,120 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+export function generateText(backendUrl, messages, tools, token) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const response = yield fetch(`${backendUrl}/ai/llm`, {
+            method: 'POST',
+            body: JSON.stringify({ messages, tools }),
+            headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }
+        });
+        return yield response.json();
+    });
+}
+export function streamChatGPT(backendUrl, messages, tools, onResponse, token) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const messageId = Math.random().toString(36).substring(3);
+        let currentMessages = [...messages];
+        while (true) {
+            const messagesForApi = currentMessages.map((_a) => {
+                var { id } = _a, rest = __rest(_a, ["id"]);
+                return rest;
+            });
+            const response = yield fetch(`${backendUrl}/ai/llm`, {
+                method: 'POST',
+                body: JSON.stringify({ messages: messagesForApi, tools, stream: true }),
+                headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }
+            });
+            if (!response.body) {
+                console.error('No response body.');
+                return;
+            }
+            const reader = response.body.getReader();
+            const decoder = new TextDecoder('utf-8');
+            let content = "";
+            let done = false;
+            let toolInvocations = [];
+            let finishReason = "";
+            while (!done) {
+                const { value, done: readerDone } = yield reader.read();
+                if (value) {
+                    const chunk = decoder.decode(value, { stream: true });
+                    const lines = chunk.split('\n').filter(line => line.trim() !== '');
+                    for (const line of lines) {
+                        const command = line.substring(0, 1);
+                        if (command === '0') {
+                            const data = line.substring(3, line.length - 1);
+                            content += data;
+                            onResponse(messageId, content.replace(/\\n/g, '\n').replace(/\\+"/g, '"'), false);
+                        }
+                        else if (command === 'd' || command === 'e') {
+                            const eventData = JSON.parse(line.substring(2));
+                            finishReason = eventData.finishReason;
+                            done = true;
+                            break;
+                        }
+                        else if (command === '9') {
+                            const toolInvocation = JSON.parse(line.substring(2));
+                            toolInvocations.push(toolInvocation);
+                        }
+                    }
+                }
+                if (readerDone) {
+                    done = true;
+                }
+            }
+            if (content || toolInvocations.length > 0) {
+                currentMessages.push({
+                    id: messageId,
+                    role: "assistant",
+                    content: content,
+                    toolCalls: toolInvocations.length > 0 ? toolInvocations : undefined,
+                });
+            }
+            if (finishReason !== 'tool-calls') {
+                onResponse(messageId, content.replace(/\\n/g, '\n'), true, toolInvocations);
+                return;
+            }
+            const toolResults = [];
+            for (const toolInvocation of toolInvocations) {
+                const tool = tools.find(t => t.name === toolInvocation.toolName);
+                if (tool && tool.execute) {
+                    try {
+                        const result = yield tool.execute(toolInvocation.args);
+                        toolResults.push({
+                            id: Math.random().toString(36).substring(3),
+                            role: "user",
+                            content: `Tool '${toolInvocation.toolName}' returned: ${JSON.stringify(result)}`,
+                        });
+                    }
+                    catch (error) {
+                        console.error(`Error executing tool ${toolInvocation.toolName}:`, error);
+                        toolResults.push({
+                            id: Math.random().toString(36).substring(3),
+                            role: "user",
+                            content: `Tool '${toolInvocation.toolName}' failed with error: ${error}`,
+                        });
+                    }
+                }
+            }
+            currentMessages.push(...toolResults);
+        }
+    });
+}

package/dist/{controller → core/controller}/ObjectController.d.ts

@@ -8,7 +8,15 @@ interface ObjectToolParameter {
     type: ObjectToolParameterType;
     description?: string;
     enum?: string[];
+    optional?: boolean;
 }
+/**
+ * The tools that the AI can use.
+ *
+ * The key is the name of the tool.
+ * The value is the parameter of the tool.
+ *
+ */
 export type ObjectTool = {
     [key: string]: ObjectToolParameter;
 };

package/dist/{controller → core/controller}/SettingsController.d.ts

@@ -1,6 +1,14 @@
 import { SupabaseClient } from "@supabase/supabase-js";
-import { LanguageLevel } from "../utils/difficultyConverter";
-import { Language } from "../utils/Language";
+import { LanguageLevel } from "../../utils/difficultyConverter";
+import { Language } from "../../utils/Language";
+export interface Buddy {
+    id: string;
+    name: string;
+    description: string;
+    avatarUrl: string;
+    voiceId: string;
+    aiPersonality: string;
+}
 export interface UserInfo {
     skill_level_reading: LanguageLevel;
     skill_level_writing: LanguageLevel;
@@ -10,13 +18,14 @@ export interface UserInfo {
     skill_level_understanding: LanguageLevel;
     goal_longterm: string;
     goal_weekly: string;
-    study_buddy: string;
+    study_buddy: Buddy;
     story_genre: string;
     study_duration: number;
     mother_tongue: Language;
     motivation_type: string;
     onboarding_completed: boolean;
     context_menu_on_select: boolean;
+    user_name?: string;
 }
 export declare class SettingsController {
     private pluginId;
@@ -24,7 +33,6 @@ export declare class SettingsController {
     constructor(supabase: SupabaseClient, pluginId: string);
     private fetchSettings;
     setSettings(settings: any): Promise<void>;
-    getUserInfo(): Promise<UserInfo>;
     /**
      * Get the settings for the plugin. T can be any type of settings, UserSettings or SystemSettings.
      * @param defaultSettings The default settings to use if no settings are found.

package/dist/{controller → core/controller}/SettingsController.js

@@ -26,31 +26,6 @@ export class SettingsController {
             yield this.supabase.from("plugin_settings").upsert({ plugin_id: this.pluginId, settings });
         });
     }
-    getUserInfo() {
-        return __awaiter(this, void 0, void 0, function* () {
-            const { data } = yield this.supabase.from("profiles").select("*");
-            if (!data || data.length === 0) {
-                return {
-                    mother_tongue: "en",
-                    skill_level_listening: "Pre-A1",
-                    skill_level_reading: "Pre-A1",
-                    skill_level_speaking: "Pre-A1",
-                    skill_level_writing: "Pre-A1",
-                    skill_level_understanding: "Pre-A1",
-                    skill_level_grammar: "Pre-A1",
-                    goal_longterm: "",
-                    goal_weekly: "",
-                    study_buddy: "clarence",
-                    story_genre: "adventure",
-                    study_duration: 30,
-                    motivation_type: "self-motivated",
-                    onboarding_completed: false,
-                    context_menu_on_select: false,
-                };
-            }
-            return data[0].settings;
-        });
-    }
     /**
      * Get the settings for the plugin. T can be any type of settings, UserSettings or SystemSettings.
      * @param defaultSettings The default settings to use if no settings are found.

package/dist/{controller → core/controller}/SharedContentController.d.ts

@@ -1,5 +1,5 @@
 import { SupabaseClient } from '@supabase/supabase-js';
-import { RimoriClient } from "../plugin/RimoriClient";
+import { RimoriClient } from "../../plugin/RimoriClient";
 import { ObjectRequest } from "./ObjectController";
 export interface SharedContentObjectRequest extends ObjectRequest {
     fixedProperties?: Record<string, string | number | boolean>;

package/dist/{controller → core/controller}/SharedContentController.js

@@ -46,14 +46,14 @@ export class SharedContentController {
             // generate new assignments
             const fullInstructions = yield this.getGeneratorInstructions(contentType, generatorInstructions, filter);
             console.log('fullInstructions:', fullInstructions);
-            const instructions = yield this.rimoriClient.llm.getObject(fullInstructions);
+            const instructions = yield this.rimoriClient.ai.getObject(fullInstructions);
             console.log('instructions:', instructions);
             const { data: newAssignment, error: insertError } = yield this.supabase.from("shared_content").insert({
                 private: privateTopic,
                 content_type: contentType,
-                topic: instructions.topic,
+                title: instructions.title,
                 keywords: instructions.keywords.map(({ text }) => text),
-                data: Object.assign(Object.assign(Object.assign({}, instructions), { topic: undefined, keywords: undefined }), generatorInstructions.fixedProperties),
+                data: Object.assign(Object.assign(Object.assign({}, instructions), { title: undefined, keywords: undefined }), generatorInstructions.fixedProperties),
             }).select();
             if (insertError) {
                 console.error('error inserting new assignment:', insertError);
@@ -67,7 +67,7 @@ export class SharedContentController {
             const completedTopics = yield this.getCompletedTopics(contentType, filter);
             generatorInstructions.instructions += `
     The following topics are already taken: ${completedTopics.join(', ')}`;
-            generatorInstructions.tool.topic = {
+            generatorInstructions.tool.title = {
                 type: "string",
                 description: "What the topic is about. Short. ",
             };

package/dist/core/core.d.ts

@@ -0,0 +1,13 @@
+export * from "../fromRimori/PluginTypes";
+export * from "../plugin/PluginController";
+export * from "../plugin/RimoriClient";
+export * from "../utils/difficultyConverter";
+export * from "../utils/Language";
+export * from "../utils/PluginUtils";
+export * from "../worker/WorkerSetup";
+export { EventBusMessage } from "../fromRimori/EventBus";
+export { Buddy, UserInfo } from "./controller/SettingsController";
+export { SharedContent } from "./controller/SharedContentController";
+export { Message, OnLLMResponse, ToolInvocation } from "./controller/AIController";
+export { MacroAccomplishmentPayload, MicroAccomplishmentPayload } from "../plugin/AccomplishmentHandler";
+export { Tool } from "../fromRimori/PluginTypes";

package/dist/core/core.js

@@ -0,0 +1,8 @@
+// Core functionality exports
+export * from "../fromRimori/PluginTypes";
+export * from "../plugin/PluginController";
+export * from "../plugin/RimoriClient";
+export * from "../utils/difficultyConverter";
+export * from "../utils/Language";
+export * from "../utils/PluginUtils";
+export * from "../worker/WorkerSetup";

package/dist/{plugin/fromRimori → fromRimori}/EventBus.d.ts

@@ -52,7 +52,7 @@ export declare class EventBusHandler {
      * @param topics - The topic of the event.
      * @param handler - The handler to be called when the event is emitted.
      * @param ignoreSender - The senders to ignore.
-     * @returns The ids of the listeners.
+     * @returns An EventListener object containing an off() method to unsubscribe the listeners.
      */
     on<T = EventPayload>(topics: string | string[], handler: EventHandler<T>, ignoreSender?: string[]): EventListener;
     /**
@@ -60,9 +60,9 @@ export declare class EventBusHandler {
      * @param sender - The sender of the event.
      * @param topic - The topic of the event.
      * @param handler - The handler to be called when the event is received. The handler returns the data to be emitted. Can be a static object or a function.
-     * @returns The ids of the listeners.
+     * @returns An EventListener object containing an off() method to unsubscribe the listeners.
      */
-    respond(sender: string, topic: string, handler: EventPayload | ((data: EventBusMessage) => EventPayload | Promise<EventPayload>)): EventListener;
+    respond(sender: string, topic: string | string[], handler: EventPayload | ((data: EventBusMessage) => EventPayload | Promise<EventPayload>)): EventListener;
     /**
      * Subscribes to an event on the event bus. The handler will be called once and then removed.
      * @param topic - The topic of the event.

package/dist/{plugin/fromRimori → fromRimori}/EventBus.js

@@ -90,7 +90,7 @@ export class EventBusHandler {
      * @param topics - The topic of the event.
      * @param handler - The handler to be called when the event is emitted.
      * @param ignoreSender - The senders to ignore.
-     * @returns The ids of the listeners.
+     * @returns An EventListener object containing an off() method to unsubscribe the listeners.
      */
     on(topics, handler, ignoreSender = []) {
         const ids = this.toArray(topics).map(topic => {
@@ -117,16 +117,33 @@ export class EventBusHandler {
      * @param sender - The sender of the event.
      * @param topic - The topic of the event.
      * @param handler - The handler to be called when the event is received. The handler returns the data to be emitted. Can be a static object or a function.
-     * @returns The ids of the listeners.
+     * @returns An EventListener object containing an off() method to unsubscribe the listeners.
      */
     respond(sender, topic, handler) {
-        const listener = this.on(topic, (data) => __awaiter(this, void 0, void 0, function* () {
-            const response = typeof handler === "function" ? yield handler(data) : handler;
-            this.emit(sender, topic, response, data.eventId);
-        }), [sender]);
-        this.logIfDebug(`Added respond listener ` + sender + " to topic " + topic, { listener, sender });
+        const topics = Array.isArray(topic) ? topic : [topic];
+        const listeners = topics.map(topic => {
+            const blackListedEventIds = [];
+            //To allow event communication inside the same plugin the sender needs to be ignored but the events still need to be checked for the same event just reaching the subscriber to prevent infinite loops
+            const finalIgnoreSender = !topic.startsWith("self.") ? [sender] : [];
+            const listener = this.on(topic, (data) => __awaiter(this, void 0, void 0, function* () {
+                if (blackListedEventIds.includes(data.eventId)) {
+                    // console.log("BLACKLISTED EVENT ID", data.eventId);
+                    return;
+                }
+                blackListedEventIds.push(data.eventId);
+                if (blackListedEventIds.length > 20) {
+                    blackListedEventIds.shift();
+                }
+                const response = typeof handler === "function" ? yield handler(data) : handler;
+                this.emit(sender, topic, response, data.eventId);
+            }), finalIgnoreSender);
+            this.logIfDebug(`Added respond listener ` + sender + " to topic " + topic, { listener, sender });
+            return {
+                off: () => listener.off()
+            };
+        });
         return {
-            off: () => listener.off()
+            off: () => listeners.forEach(listener => listener.off())
         };
     }
     /**