cactus-react-native 0.2.3 → 0.2.5

package/src/lm.ts

@@ -1,4 +1,6 @@
 import { initLlama, LlamaContext } from './index'
+// @ts-ignore
+import { Platform } from 'react-native'
 import type {
   ContextParams,
   CompletionParams,
@@ -7,8 +9,10 @@ import type {
   EmbeddingParams,
   NativeEmbeddingResult,
 } from './index'
+
 import { Telemetry } from './telemetry'
 import { setCactusToken, getVertexAIEmbedding } from './remote'
+import { ConversationHistoryManager } from './chat'
 
 interface CactusLMReturn {
   lm: CactusLM | null
@@ -16,10 +20,20 @@ interface CactusLMReturn {
 }
 
 export class CactusLM {
-  private context: LlamaContext
+  protected context: LlamaContext
+  protected conversationHistoryManager: ConversationHistoryManager
+
+  // the initPromise enables a "async singleton" initialization pattern which
+  // protects against a race condition in the event of multiple init attempts
+  private static _initCache: Map<string, Promise<CactusLMReturn>> = new Map();
+
+  private static getCacheKey(params: ContextParams, cactusToken?: string, retryOptions?: { maxRetries?: number; delayMs?: number }): string {
+    return JSON.stringify({ params, cactusToken, retryOptions });
+  }
 
-  private constructor(context: LlamaContext) {
+  protected constructor(context: LlamaContext) {
     this.context = context
+    this.conversationHistoryManager = new ConversationHistoryManager()
   }
 
   static async init(
@@ -28,72 +42,108 @@ export class CactusLM {
     cactusToken?: string,
     retryOptions?: { maxRetries?: number; delayMs?: number },
   ): Promise<CactusLMReturn> {
+
     if (cactusToken) {
       setCactusToken(cactusToken);
     }
 
-    const maxRetries = retryOptions?.maxRetries ?? 3;
-    const delayMs = retryOptions?.delayMs ?? 1000;
-
-    const configs = [
-      params,
-      { ...params, n_gpu_layers: 0 } 
-    ];
-
-    const sleep = (ms: number): Promise<void> => {
-      return new Promise(resolve => {
-        const start = Date.now();
-        const wait = () => {
-          if (Date.now() - start >= ms) {
-            resolve();
-          } else {
-            Promise.resolve().then(wait);
-          }
-        };
-        wait();
-      });
-    };
+    const key = CactusLM.getCacheKey(params, cactusToken, retryOptions);
+    if (CactusLM._initCache.has(key)) {
+      // concurrent initialization calls with the same params all get the same cached Promise
+      return CactusLM._initCache.get(key)!;
+    }
 
-    for (const config of configs) {
-      let lastError: Error | null = null;
-      
-      for (let attempt = 1; attempt <= maxRetries; attempt++) {
-        try {
-          const context = await initLlama(config, onProgress);
-          return { lm: new CactusLM(context), error: null };
-        } catch (e) {
-          lastError = e as Error;
-          const isLastConfig = configs.indexOf(config) === configs.length - 1;
-          const isLastAttempt = attempt === maxRetries;
-          
-          Telemetry.error(e as Error, {
-            n_gpu_layers: config.n_gpu_layers ?? null,
-            n_ctx: config.n_ctx ?? null,
-            model: config.model ?? null,
-          });
-          
-          if (!isLastAttempt) {
-            const delay = delayMs * Math.pow(2, attempt - 1);
-            await sleep(delay);
-          } else if (!isLastConfig) {
-            break;
+    const initPromise = (async () => {
+      const maxRetries = retryOptions?.maxRetries ?? 3;
+      const delayMs = retryOptions?.delayMs ?? 1000;
+
+      const configs = [
+        params,
+        { ...params, n_gpu_layers: 0 }
+      ];
+
+      const sleep = (ms: number): Promise<void> => {
+        return new Promise(resolve => {
+          const start = Date.now();
+          const wait = () => {
+            if (Date.now() - start >= ms) {
+              resolve();
+            } else {
+              Promise.resolve().then(wait);
+            }
+          };
+          wait();
+        });
+      };
+
+      for (const config of configs) {
+        let lastError: Error | null = null;
+
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+          try {
+            const context = await initLlama(config, onProgress);
+            return { lm: new CactusLM(context), error: null };
+          } catch (e) {
+            lastError = e as Error;
+            const isLastConfig = configs.indexOf(config) === configs.length - 1;
+            const isLastAttempt = attempt === maxRetries;
+
+            Telemetry.error(e as Error, {
+              n_gpu_layers: config.n_gpu_layers ?? null,
+              n_ctx: config.n_ctx ?? null,
+              model: config.model ?? null,
+            });
+
+            if (!isLastAttempt) {
+              const delay = delayMs * Math.pow(2, attempt - 1);
+              await sleep(delay);
+            } else if (!isLastConfig) {
+              break;
+            }
           }
         }
+
+        if (configs.indexOf(config) === configs.length - 1 && lastError) {
+          return { lm: null, error: lastError };
+        }
       }
-      
-      if (configs.indexOf(config) === configs.length - 1 && lastError) {
-        return { lm: null, error: lastError };
-      }
+      return { lm: null, error: new Error('Failed to initialize CactusLM after all retries') };
+    })();
+
+    CactusLM._initCache.set(key, initPromise);
+
+    const result = await initPromise;
+    if (result.error) {
+      CactusLM._initCache.delete(key); // Reset on failure to allow retries
     }
-    return { lm: null, error: new Error('Failed to initialize CactusLM after all retries') };
+    return result;
   }
 
-  async completion(
+  completion = async (
     messages: CactusOAICompatibleMessage[],
     params: CompletionParams = {},
     callback?: (data: any) => void,
-  ): Promise<NativeCompletionResult> {
-    return await this.context.completion({ messages, ...params }, callback);
+  ): Promise<NativeCompletionResult> => {
+    const { newMessages, requiresReset } =
+      this.conversationHistoryManager.processNewMessages(messages);
+
+    if (requiresReset) {
+      this.context?.rewind();
+      this.conversationHistoryManager.reset();
+    }
+
+    if (newMessages.length === 0) {
+      console.warn('No messages to complete!');
+    }
+
+    const result = await this.context.completion({ messages: newMessages, ...params }, callback);
+
+    this.conversationHistoryManager.update(newMessages, {
+      role: 'assistant',
+      content: result.content,
+    });
+
+    return result;
   }
 
   async embedding(
@@ -136,23 +186,27 @@ export class CactusLM {
     return result;
   }
 
-  private async _handleLocalEmbedding(text: string, params?: EmbeddingParams): Promise<NativeEmbeddingResult> {
+  protected async _handleLocalEmbedding(text: string, params?: EmbeddingParams): Promise<NativeEmbeddingResult> {
     return this.context.embedding(text, params);
   }
 
-  private async _handleRemoteEmbedding(text: string): Promise<NativeEmbeddingResult> {
+  protected async _handleRemoteEmbedding(text: string): Promise<NativeEmbeddingResult> {
     const embeddingValues = await getVertexAIEmbedding(text);
     return {
       embedding: embeddingValues,
     };
   }
 
-  async rewind(): Promise<void> {
-    // @ts-ignore
+  rewind = async (): Promise<void> => {
     return this.context?.rewind()
   }
 
   async release(): Promise<void> {
     return this.context.release()
   }
+
+  async stopCompletion(): Promise<void> {
+    return await this.context.stopCompletion()
+  }
+
 } 

package/src/tools.ts

@@ -56,22 +56,18 @@ export class Tools {
 
 export async function parseAndExecuteTool(result: NativeCompletionResult, tools: Tools): Promise<{toolCalled: boolean, toolName?: string, toolInput?: any, toolOutput?: any}> {
   if (!result.tool_calls || result.tool_calls.length === 0) {
-      // console.log('No tool calls found');
       return {toolCalled: false};
   }
   
   try {
       const toolCall = result.tool_calls[0];
       if (!toolCall) {
-        // console.log('No tool call found');
         return {toolCalled: false};
       }
       const toolName = toolCall.function.name;
       const toolInput = JSON.parse(toolCall.function.arguments);
       
-      // console.log('Calling tool:', toolName, toolInput);
       const toolOutput = await tools.execute(toolName, toolInput);
-      // console.log('Tool called result:', toolOutput);
       
       return {
           toolCalled: true,
@@ -80,7 +76,6 @@ export async function parseAndExecuteTool(result: NativeCompletionResult, tools:
           toolOutput
       };
   } catch (error) {
-      // console.error('Error parsing tool call:', error);
       return {toolCalled: false};
   }
 }

package/src/tts.ts

@@ -31,10 +31,7 @@ export class CactusTTS {
       speakerJsonStr,
       textToSpeak,
     )
-    // This part is simplified. In a real scenario, the tokens from 
-    // the main model would be generated and passed to decodeAudioTokens.
-    // For now, we are assuming a direct path which may not be fully functional
-    // without the main model's token output for TTS.
+    // To-DO: Fix
     const tokens = (await this.context.tokenize(formatted_prompt)).tokens
     return decodeAudioTokens(this.context.id, tokens)
   }

package/src/vlm.ts

@@ -3,15 +3,15 @@ import {
   initMultimodal,
   multimodalCompletion,
   LlamaContext,
+  type ContextParams,
+  type CompletionParams,
+  type CactusOAICompatibleMessage,
+  type NativeCompletionResult,
 } from './index'
-import type {
-  ContextParams,
-  CompletionParams,
-  CactusOAICompatibleMessage,
-  NativeCompletionResult,
-} from './index'
+
 import { Telemetry } from './telemetry'
 import { setCactusToken, getTextCompletion, getVisionCompletion } from './remote'
+import { ConversationHistoryManager } from './chat'
 
 interface CactusVLMReturn {
   vlm: CactusVLM | null
@@ -29,43 +29,101 @@ export type VLMCompletionParams = Omit<CompletionParams, 'prompt'> & {
 
 export class CactusVLM {
   private context: LlamaContext
-  
+  protected conversationHistoryManager: ConversationHistoryManager
+
+  // see CactusLM for detailed docs
+  private static _initCache: Map<string, Promise<CactusVLMReturn>> = new Map();
+
+  private static getCacheKey(params: VLMContextParams, cactusToken?: string, retryOptions?: { maxRetries?: number; delayMs?: number }): string {
+    return JSON.stringify({ params, cactusToken, retryOptions });
+  }
+
   private constructor(context: LlamaContext) {
     this.context = context
+    this.conversationHistoryManager = new ConversationHistoryManager()
   }
 
   static async init(
     params: VLMContextParams,
     onProgress?: (progress: number) => void,
     cactusToken?: string,
+    retryOptions?: { maxRetries?: number; delayMs?: number },
   ): Promise<CactusVLMReturn> {
     if (cactusToken) {
       setCactusToken(cactusToken);
     }
 
-    const configs = [
-      params,
-      { ...params, n_gpu_layers: 0 } 
-    ];
+    const key = CactusVLM.getCacheKey(params, cactusToken, retryOptions);
+    if (CactusVLM._initCache.has(key)) {
+      return CactusVLM._initCache.get(key)!;
+    }
 
-    for (const config of configs) {
-      try {
-        const context = await initLlama(config, onProgress)
-        await initMultimodal(context.id, params.mmproj, false)
-        return {vlm: new CactusVLM(context), error: null}
-      } catch (e) {
-        Telemetry.error(e as Error, {
-          n_gpu_layers: config.n_gpu_layers ?? null,
-          n_ctx: config.n_ctx ?? null,
-          model: config.model ?? null,
+    const initPromise = (async () => {
+      const maxRetries = retryOptions?.maxRetries ?? 3;
+      const delayMs = retryOptions?.delayMs ?? 1000;
+
+      const configs = [
+        params,
+        { ...params, n_gpu_layers: 0 } 
+      ];
+
+      const sleep = (ms: number): Promise<void> => {
+        return new Promise(resolve => {
+          const start = Date.now();
+          const wait = () => {
+            if (Date.now() - start >= ms) {
+              resolve();
+            } else {
+              Promise.resolve().then(wait);
+            }
+          };
+          wait();
         });
-        if (configs.indexOf(config) === configs.length - 1) {
-          return {vlm: null, error: e as Error}
+      };
+
+      for (const config of configs) {
+        let lastError: Error | null = null;
+
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+          try {
+            const context = await initLlama(config, onProgress)
+            await initMultimodal(context.id, params.mmproj, false)
+            return {vlm: new CactusVLM(context), error: null}
+          } catch (e) {
+            lastError = e as Error;
+            const isLastConfig = configs.indexOf(config) === configs.length - 1;
+            const isLastAttempt = attempt === maxRetries;
+
+            Telemetry.error(e as Error, {
+              n_gpu_layers: config.n_gpu_layers ?? null,
+              n_ctx: config.n_ctx ?? null,
+              model: config.model ?? null,
+            });
+
+            if (!isLastAttempt) {
+              const delay = delayMs * Math.pow(2, attempt - 1);
+              await sleep(delay);
+            } else if (!isLastConfig) {
+              break;
+            }
+          }
+        }
+
+        if (configs.indexOf(config) === configs.length - 1 && lastError) {
+          return {vlm: null, error: lastError}
         }
       }
-    }
 
-    return {vlm: null, error: new Error('Failed to initialize CactusVLM')}
+      return {vlm: null, error: new Error('Failed to initialize CactusVLM')}
+    })();
+
+    CactusVLM._initCache.set(key, initPromise);
+
+    const result = await initPromise;
+    if (result.error) {
+      CactusVLM._initCache.delete(key); 
+    }
+    return result;
   }
 
   async completion(
@@ -111,26 +169,47 @@ export class CactusVLM {
     return result;
   }
 
-  private async _handleLocalCompletion(
+  private _handleLocalCompletion = async(
     messages: CactusOAICompatibleMessage[],
     params: VLMCompletionParams,
     callback?: (data: any) => void,
-  ): Promise<NativeCompletionResult> {
+  ): Promise<NativeCompletionResult> => {
+    const { newMessages, requiresReset } =
+      this.conversationHistoryManager.processNewMessages(messages);
+
+    if (requiresReset) {
+      this.context?.rewind();
+      this.conversationHistoryManager.reset();
+    }
+
+    if (newMessages.length === 0) {
+      console.warn('No messages to complete!');
+    }
+
+    let result: NativeCompletionResult;
+
     if (params.images && params.images.length > 0) {
-      const formattedPrompt = await this.context.getFormattedChat(messages)
+      const formattedPrompt = await this.context.getFormattedChat(newMessages)
       const prompt =
         typeof formattedPrompt === 'string'
           ? formattedPrompt
           : formattedPrompt.prompt
-      return await multimodalCompletion(
+      result = await multimodalCompletion(
         this.context.id,
         prompt,
         params.images,
         { ...params, prompt, emit_partial_completion: !!callback },
       )
     } else {
-      return await this.context.completion({ messages, ...params }, callback)
+      result = await this.context.completion({ messages: newMessages, ...params }, callback)
     }
+
+    this.conversationHistoryManager.update(newMessages, {
+      role: 'assistant',
+      content: result.content || result.text,
+    });
+
+    return result;
   }
 
   private async _handleRemoteCompletion(
@@ -181,11 +260,14 @@ export class CactusVLM {
   }
 
   async rewind(): Promise<void> {
-    // @ts-ignore
     return this.context?.rewind()
   }
 
   async release(): Promise<void> {
     return this.context.release()
   }
+
+  async stopCompletion(): Promise<void> {
+    return await this.context.stopCompletion()
+  }
 }