@rimori/client 1.3.1 → 1.4.0

package/src/cli/types/DatabaseTypes.ts

@@ -1,10 +1,16 @@
-
 // Database table structure definitions
 
 /**
  * Supported database column data types for table schema definitions.
  */
-type DbColumnType = 'decimal' | 'integer' | 'text' | 'boolean' | 'json' | 'timestamp' | 'uuid';
+type DbColumnType =
+  | 'decimal'
+  | 'integer'
+  | 'text'
+  | 'boolean'
+  | 'json'
+  | 'timestamp'
+  | 'uuid';
 
 /**
  * Foreign key relationship configuration with cascade delete support.
@@ -45,15 +51,15 @@ export interface DbColumnDefinition {
   // primary_key?: boolean;
   /** Restrictions for the column. If the column is restricted, the permission is further restricted. E.g. if the column is restricted to user, then the user can only read the column if they have the right permission.
    * Example: Denying users to update the column, but allowing the moderator to update the column.
-  */
+   */
   restrict?: {
     /** Restrictions for the user */
-    user: Partial<Omit<DbPermissionDefinition, 'delete'>>,
+    user: Partial<Omit<DbPermissionDefinition, 'delete'>>;
     /** Restrictions for the moderator */
-    moderator?: Partial<Omit<DbPermissionDefinition, 'delete'>>,
+    moderator?: Partial<Omit<DbPermissionDefinition, 'delete'>>;
     /** Restrictions for the maintainer */
     // maintainer?: Partial<DbPermissionDefinition>,
-  }
+  };
 }
 
 /**
@@ -80,8 +86,8 @@ export interface DbTableDefinition {
   description: string;
   /** Permissions for the table */
   permissions: {
-    user: DbPermissionDefinition,
-    moderator?: DbPermissionDefinition,
+    user: DbPermissionDefinition;
+    moderator?: DbPermissionDefinition;
     // maintainer?: DbPermissionDefinition,
   };
   /** Column definitions for the table */
@@ -98,7 +104,7 @@ export interface DbTableDefinition {
  *
  * Defines the permissions for a database table.
  */
-export type DbPermission = "NONE" | "OWN" | "ALL";
+export type DbPermission = 'NONE' | 'OWN' | 'ALL';
 
 /**
  * Permission definition for a database table.
@@ -114,4 +120,5 @@ export interface DbPermissionDefinition {
 /**
  * Full table definition that includes automatically generated fields.
  */
-export type FullTable<T extends Record<string, DbColumnDefinition>> = T & BaseTableStructure;
+export type FullTable<T extends Record<string, DbColumnDefinition>> = T &
+  BaseTableStructure;

package/src/components/ai/Avatar.tsx

@@ -44,6 +44,7 @@ export function Avatar({
   }, [isLoading]);
 
   useEffect(() => {
+    if (!voiceId) return; //at the beginning when being mounted the voiceId is undefined
     sender.setOnLoudnessChange((value) => event.emit('self.avatar.triggerLoudness', { loudness: value }));
     sender.setOnEndOfSpeech(() => setAgentReplying(false));
 
@@ -58,13 +59,13 @@ export function Avatar({
     } else if (autoStartConversation.userMessage) {
       append([{ role: 'user', content: autoStartConversation.userMessage, id: messages.length.toString() }]);
     }
-  }, [autoStartConversation]);
+  }, [autoStartConversation, voiceId]);
 
   useEffect(() => {
     if (lastMessage?.role === 'assistant') {
       sender.handleNewText(lastMessage.content, isLoading);
       if (lastMessage.toolCalls) {
-        console.log("unlocking mic",lastMessage)
+        // console.log("unlocking mic", lastMessage)
         setAgentReplying(false);
         setIsProcessingMessage(false);
       }

package/src/components/ai/EmbeddedAssistent/TTS/Player.ts

@@ -1,198 +1,198 @@
 export class ChunkedAudioPlayer {
 
-    private audioContext!: AudioContext;
-    private chunkQueue: ArrayBuffer[] = [];
-    private isPlaying = false;
-    private analyser!: AnalyserNode;
-    private dataArray!: Uint8Array;
-    private shouldMonitorLoudness = true;
-    private isMonitoring = false;
-    private handle = 0;
-    private volume = 1.0;
-    private loudnessCallback: (value: number) => void = () => { };
-    private currentIndex = 0;
-    private startedPlaying = false;
-    private onEndOfSpeech: () => void = () => { };
-
-    constructor() {
-        this.init();
+  private audioContext!: AudioContext;
+  private chunkQueue: ArrayBuffer[] = [];
+  private isPlaying = false;
+  private analyser!: AnalyserNode;
+  private dataArray!: Uint8Array<ArrayBuffer>;
+  private shouldMonitorLoudness = true;
+  private isMonitoring = false;
+  private handle = 0;
+  private volume = 1.0;
+  private loudnessCallback: (value: number) => void = () => { };
+  private currentIndex = 0;
+  private startedPlaying = false;
+  private onEndOfSpeech: () => void = () => { };
+
+  constructor() {
+    this.init();
+  }
+
+  private init(): void {
+    this.audioContext = new AudioContext();
+    this.analyser = this.audioContext.createAnalyser();
+    this.analyser.fftSize = 256; // Set the FFT size (smaller values provide faster updates, larger ones give better resolution)
+    const bufferLength = this.analyser.frequencyBinCount;
+    this.dataArray = new Uint8Array(bufferLength); // Array to hold frequency data
+  }
+
+  public setOnLoudnessChange(callback: (value: number) => void) {
+    this.loudnessCallback = callback;
+  }
+
+  public setVolume(volume: number) {
+    this.volume = volume;
+  }
+
+  public async addChunk(chunk: ArrayBuffer, position: number): Promise<void> {
+    console.log('Adding chunk', position, chunk);
+    this.chunkQueue[position] = chunk;
+    // console.log("received chunk", {
+    //     chunkQueue: this.chunkQueue.length,
+    //     isPlaying: this.isPlaying,
+    // })
+
+    if (position === 0 && !this.startedPlaying) {
+      this.startedPlaying = true;
+      this.playChunks();
     }
-
-    private init(): void {
-        this.audioContext = new AudioContext();
-        this.analyser = this.audioContext.createAnalyser();
-        this.analyser.fftSize = 256; // Set the FFT size (smaller values provide faster updates, larger ones give better resolution)
-        const bufferLength = this.analyser.frequencyBinCount;
-        this.dataArray = new Uint8Array(bufferLength); // Array to hold frequency data
-    }
-
-    public setOnLoudnessChange(callback: (value: number) => void) {
-        this.loudnessCallback = callback;
+  }
+
+  private playChunks(): void {
+    // console.log({ isPlaying: this.isPlaying });
+    if (this.isPlaying) return;
+    if (!this.chunkQueue[this.currentIndex]) {
+      // wait until the correct chunk arrives
+      setTimeout(() => this.playChunks(), 10);
     }
+    this.isPlaying = true;
 
-    public setVolume(volume: number) {
-        this.volume = volume;
-    }
-
-    public async addChunk(chunk: ArrayBuffer, position: number): Promise<void> {
-        console.log('Adding chunk', position, chunk);
-        this.chunkQueue[position] = chunk;
-        // console.log("received chunk", {
-        //     chunkQueue: this.chunkQueue.length,
-        //     isPlaying: this.isPlaying,
-        // })
-
-        if (position === 0 && !this.startedPlaying) {
-            this.startedPlaying = true;
+    this.playChunk(this.chunkQueue[this.currentIndex]).then(() => {
+      this.isPlaying = false;
+      this.currentIndex++;
+      if (this.chunkQueue[this.currentIndex]) {
+        this.shouldMonitorLoudness = true;
+        this.playChunks();
+      } else {
+        // console.log('Playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
+        setTimeout(() => {
+          // console.log('Check again if really playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
+          if (this.chunkQueue.length > this.currentIndex) {
             this.playChunks();
-        }
+          } else {
+            this.startedPlaying = false;
+            this.shouldMonitorLoudness = false;
+          }
+        }, 1000);
+      }
+    });
+  }
+
+  public stopPlayback(): void {
+    // console.log('Stopping playback');
+    // Implement logic to stop the current playback
+    this.isPlaying = false;
+    this.chunkQueue = [];
+    this.startedPlaying = false;
+    this.shouldMonitorLoudness = false;
+    cancelAnimationFrame(this.handle);
+  }
+
+  private playChunk(chunk: ArrayBuffer): Promise<void> {
+    // console.log({queue: this.chunkQueue})
+    if (!chunk) {
+      return Promise.resolve();
     }
 
-    private playChunks(): void {
-        // console.log({ isPlaying: this.isPlaying });
-        if (this.isPlaying) return;
-        if (!this.chunkQueue[this.currentIndex]) {
-            // wait until the correct chunk arrives
-            setTimeout(() => this.playChunks(), 10);
+    // console.log('Playing chunk', chunk);
+    return new Promise((resolve) => {
+      const source = this.audioContext.createBufferSource();
+      this.audioContext.decodeAudioData(chunk.slice(0)).then((audioBuffer) => {
+        source.buffer = audioBuffer;
+
+        // Create a GainNode for volume control
+        const gainNode = this.audioContext.createGain();
+        gainNode.gain.value = this.volume;
+
+        // Connect the source to the GainNode, then to the analyser node, then to the destination (speakers)
+        source.connect(gainNode);
+        gainNode.connect(this.analyser);
+        this.analyser.connect(this.audioContext.destination);
+
+        source.start(0);
+        // console.log('Playing chunk', this.currentIndex);
+        gainNode.gain.value = this.volume;
+        source.onended = () => {
+          // console.log('Chunk ended');
+          resolve();
+        };
+
+        // Start monitoring loudness only once
+        if (!this.isMonitoring) {
+          this.isMonitoring = true;
+          this.shouldMonitorLoudness = true;
+          this.monitorLoudness();
         }
-        this.isPlaying = true;
-
-        this.playChunk(this.chunkQueue[this.currentIndex]).then(() => {
-            this.isPlaying = false;
-            this.currentIndex++;
-            if (this.chunkQueue[this.currentIndex]) {
-                this.shouldMonitorLoudness = true;
-                this.playChunks();
-            } else {
-                // console.log('Playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
-                setTimeout(() => {
-                    // console.log('Check again if really playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
-                    if (this.chunkQueue.length > this.currentIndex) {
-                        this.playChunks();
-                    } else {
-                        this.startedPlaying = false;
-                        this.shouldMonitorLoudness = false;
-                    }
-                }, 1000);
-            }
-        });
+      });
+    });
+  }
+
+  async playAgain(): Promise<void> {
+    console.log('Playing again');
+    if (this.chunkQueue.length > 0 && !this.isPlaying) {
+      this.playChunks();
     }
-
-    public stopPlayback(): void {
-        // console.log('Stopping playback');
-        // Implement logic to stop the current playback
-        this.isPlaying = false;
-        this.chunkQueue = [];
-        this.startedPlaying = false;
-        this.shouldMonitorLoudness = false;
-        cancelAnimationFrame(this.handle);
+  }
+
+  private monitorLoudness(): void {
+    // Stop monitoring when the flag is false
+    if (!this.shouldMonitorLoudness) {
+      // console.log('Loudness monitoring stopped.');
+      cancelAnimationFrame(this.handle);
+      this.loudnessCallback(0);
+      this.onEndOfSpeech();
+      return;
     }
 
-    private playChunk(chunk: ArrayBuffer): Promise<void> {
-        console.log({queue: this.chunkQueue})
-        if (!chunk) {
-            return Promise.resolve();
-        }
-
-        // console.log('Playing chunk', chunk);
-        return new Promise((resolve) => {
-            const source = this.audioContext.createBufferSource();
-            this.audioContext.decodeAudioData(chunk.slice(0)).then((audioBuffer) => {
-                source.buffer = audioBuffer;
-
-                // Create a GainNode for volume control
-                const gainNode = this.audioContext.createGain();
-                gainNode.gain.value = this.volume;
-
-                // Connect the source to the GainNode, then to the analyser node, then to the destination (speakers)
-                source.connect(gainNode);
-                gainNode.connect(this.analyser);
-                this.analyser.connect(this.audioContext.destination);
-
-                source.start(0);
-                // console.log('Playing chunk', this.currentIndex);
-                gainNode.gain.value = this.volume;
-                source.onended = () => {
-                    // console.log('Chunk ended');
-                    resolve();
-                };
-
-                // Start monitoring loudness only once
-                if (!this.isMonitoring) {
-                    this.isMonitoring = true;
-                    this.shouldMonitorLoudness = true;
-                    this.monitorLoudness();
-                }
-            });
-        });
-    }
+    // Get the time domain data from the analyser (this is a snapshot of the waveform)
+    this.analyser.getByteTimeDomainData(this.dataArray);
 
-    async playAgain(): Promise<void> {
-        console.log('Playing again');
-        if (this.chunkQueue.length > 0 && !this.isPlaying) {
-            this.playChunks();
-        }
+    // Calculate the RMS (root mean square) of the waveform values to get the perceived loudness
+    let sum = 0;
+    for (let i = 0; i < this.dataArray.length; i++) {
+      const value = this.dataArray[i] / 128.0 - 1.0; // Normalize between -1 and 1
+      sum += value * value;
     }
 
-    private monitorLoudness(): void {
-        // Stop monitoring when the flag is false
-        if (!this.shouldMonitorLoudness) {
-            // console.log('Loudness monitoring stopped.');
-            cancelAnimationFrame(this.handle);
-            this.loudnessCallback(0);
-            this.onEndOfSpeech();
-            return;
-        }
-
-        // Get the time domain data from the analyser (this is a snapshot of the waveform)
-        this.analyser.getByteTimeDomainData(this.dataArray);
-
-        // Calculate the RMS (root mean square) of the waveform values to get the perceived loudness
-        let sum = 0;
-        for (let i = 0; i < this.dataArray.length; i++) {
-            const value = this.dataArray[i] / 128.0 - 1.0; // Normalize between -1 and 1
-            sum += value * value;
-        }
-
-        const rms = Math.sqrt(sum / this.dataArray.length);
-
-        // Handle the case where RMS is 0 to avoid log10(0)
-        if (rms === 0) {
-            // console.log('Current loudness: Silent');
-        } else {
-            let loudnessInDb = 20 * Math.log10(rms); // Convert to dB
-            // console.log('Current loudness:' + loudnessInDb);
-            const minDb = -57;
-            const maxDb = -15;
+    const rms = Math.sqrt(sum / this.dataArray.length);
 
-            if (loudnessInDb < minDb) {
-                loudnessInDb = minDb;
-            }
-            if (loudnessInDb > maxDb) {
-                loudnessInDb = maxDb;
-            }
+    // Handle the case where RMS is 0 to avoid log10(0)
+    if (rms === 0) {
+      // console.log('Current loudness: Silent');
+    } else {
+      let loudnessInDb = 20 * Math.log10(rms); // Convert to dB
+      // console.log('Current loudness:' + loudnessInDb);
+      const minDb = -57;
+      const maxDb = -15;
 
-            const loudnessScale = ((loudnessInDb - minDb) / (maxDb - minDb)) * 100;
-            // console.log("root:corrent loudness", loudnessScale);
+      if (loudnessInDb < minDb) {
+        loudnessInDb = minDb;
+      }
+      if (loudnessInDb > maxDb) {
+        loudnessInDb = maxDb;
+      }
 
-            this.loudnessCallback(loudnessScale);
-        }
+      const loudnessScale = ((loudnessInDb - minDb) / (maxDb - minDb)) * 100;
+      // console.log("root:corrent loudness", loudnessScale);
 
-        // Call this method again at regular intervals if you want continuous loudness monitoring
-        this.handle = requestAnimationFrame(() => this.monitorLoudness());
-    }
-    public reset() {
-        // console.log('Resetting player');
-        this.stopPlayback();
-        this.currentIndex = 0;
-        this.shouldMonitorLoudness = true;
-        //reset to the beginning when the class gets initialized
-        this.isMonitoring = false;
-        this.isPlaying = false;
-        this.init();
+      this.loudnessCallback(loudnessScale);
     }
 
-    public setOnEndOfSpeech(callback: () => void) {
-        this.onEndOfSpeech = callback;
-    }
+    // Call this method again at regular intervals if you want continuous loudness monitoring
+    this.handle = requestAnimationFrame(() => this.monitorLoudness());
+  }
+  public reset() {
+    // console.log('Resetting player');
+    this.stopPlayback();
+    this.currentIndex = 0;
+    this.shouldMonitorLoudness = true;
+    //reset to the beginning when the class gets initialized
+    this.isMonitoring = false;
+    this.isPlaying = false;
+    this.init();
+  }
+
+  public setOnEndOfSpeech(callback: () => void) {
+    this.onEndOfSpeech = callback;
+  }
 }

package/src/core/controller/AIController.ts

@@ -71,10 +71,10 @@ export async function streamChatGPT(backendUrl: string, messages: Message[], too
         if (value) {
           const chunk = decoder.decode(value, { stream: true });
           buffer += chunk;
-          
+
           // Split by lines, but handle incomplete lines
           const lines = buffer.split('\n');
-          
+
           // Keep the last line in buffer if it's incomplete
           if (lines.length > 1) {
             buffer = lines.pop() || "";
@@ -82,11 +82,11 @@ export async function streamChatGPT(backendUrl: string, messages: Message[], too
 
           for (const line of lines) {
             if (line.trim() === '') continue;
-            
+
             // Handle the new streaming format
             if (line.startsWith('data: ')) {
               const dataStr = line.substring(6); // Remove 'data: ' prefix
-              
+
               // Handle [DONE] marker
               if (dataStr === '[DONE]') {
                 done = true;
@@ -95,39 +95,39 @@ export async function streamChatGPT(backendUrl: string, messages: Message[], too
 
               try {
                 const data = JSON.parse(dataStr);
-                
+
                 // Log the first message to understand the format
                 if (!content && !isToolCallMode) {
-                  console.log('First stream message received:', data);
+                  // console.log('First stream message received:', data);
                 }
-                
+
                 switch (data.type) {
                   case 'start':
                     // Stream started, no action needed
-                    console.log('Stream started');
+                    // console.log('Stream started');
                     break;
-                    
+
                   case 'start-step':
                     // Step started, no action needed
-                    console.log('Step started');
+                    // console.log('Step started');
                     break;
-                    
+
                   case 'reasoning-start':
                     // Reasoning started, no action needed
                     console.log('Reasoning started:', data.id);
                     break;
-                    
+
                   case 'reasoning-end':
                     // Reasoning ended, no action needed
                     console.log('Reasoning ended:', data.id);
                     break;
-                    
+
                   case 'text-start':
                     // Text generation started, store the ID
                     currentTextId = data.id;
                     console.log('Text generation started:', data.id);
                     break;
-                    
+
                   case 'text-delta':
                     // Text delta received, append to content
                     if (data.delta) {
@@ -135,73 +135,75 @@ export async function streamChatGPT(backendUrl: string, messages: Message[], too
                       onResponse(messageId, content, false);
                     }
                     break;
-                    
+
                   case 'text-end':
                     // Text generation ended
                     console.log('Text generation ended:', data.id);
                     break;
-                    
+
                   case 'finish-step':
                     // Step finished, no action needed
-                    console.log('Step finished');
+                    // console.log('Step finished');
                     break;
-                    
+
                   case 'finish':
                     // Stream finished
-                    console.log('Stream finished');
+                    // console.log('Stream finished');
                     done = true;
                     break;
-                    
+
                   // Additional message types that might be present in the AI library
                   case 'tool-call':
+                  case 'tool-input-available': //for now input calls should be handled the same way as tool calls
                     // Tool call initiated
                     console.log('Tool call initiated:', data);
                     isToolCallMode = true;
-                    if (data.toolCallId && data.toolName && data.args) {
+                    if (data.toolCallId && data.toolName && (data.args || data.input)) {
                       toolInvocations.push({
                         toolCallId: data.toolCallId,
                         toolName: data.toolName,
-                        args: data.args
+                        args: data.args || data.input
                       });
                     }
                     break;
-                    
+
+                  case 'tool-input-delta': //for now input calls should be handled the same way as tool calls
                   case 'tool-call-delta':
                     // Tool call delta (for streaming tool calls)
                     console.log('Tool call delta:', data);
                     break;
-                    
+
                   case 'tool-call-end':
                     // Tool call completed
                     console.log('Tool call completed:', data);
                     break;
-                    
+
                   case 'tool-result':
                     // Tool execution result
                     console.log('Tool result:', data);
                     break;
-                    
+
                   case 'error':
                     // Error occurred
                     console.error('Stream error:', data);
                     break;
-                    
+
                   case 'usage':
                     // Usage information
                     console.log('Usage info:', data);
                     break;
-                    
+
                   case 'model':
                     // Model information
                     console.log('Model info:', data);
                     break;
-                    
+
                   case 'stop':
                     // Stop signal
                     console.log('Stop signal received');
                     done = true;
                     break;
-                    
+
                   default:
                     // Unknown type, log for debugging
                     console.log('Unknown stream type:', data.type, data);
@@ -225,14 +227,14 @@ export async function streamChatGPT(backendUrl: string, messages: Message[], too
           id: messageId,
           role: "assistant",
           content: content,
-          toolCalls: toolInvocations.length > 0 ? toolInvocations: undefined,
+          toolCalls: toolInvocations.length > 0 ? toolInvocations : undefined,
         });
       }
 
       // Handle tool call scenario if tools were provided
       if (tools.length > 0 && toolInvocations.length > 0) {
         console.log('Tool calls detected, executing tools...');
-        
+
         const toolResults: Message[] = [];
         for (const toolInvocation of toolInvocations) {
           const tool = tools.find(t => t.name === toolInvocation.toolName);
@@ -254,7 +256,7 @@ export async function streamChatGPT(backendUrl: string, messages: Message[], too
             }
           }
         }
-        
+
         if (toolResults.length > 0) {
           currentMessages.push(...toolResults);
           // Continue the loop to handle the next response
@@ -273,7 +275,7 @@ export async function streamChatGPT(backendUrl: string, messages: Message[], too
 
       onResponse(messageId, content, true, toolInvocations);
       return;
-      
+
     } catch (error) {
       console.error('Error in streamChatGPT:', error);
       onResponse(messageId, `Error: ${error instanceof Error ? error.message : String(error)}`, true, []);