cactus-react-native 0.2.4 → 0.2.6

package/src/lm.ts

@@ -23,6 +23,12 @@ export class CactusLM {
   protected context: LlamaContext
   protected conversationHistoryManager: ConversationHistoryManager
 
+  private static _initCache: Map<string, Promise<CactusLMReturn>> = new Map();
+
+  private static getCacheKey(params: ContextParams, cactusToken?: string, retryOptions?: { maxRetries?: number; delayMs?: number }): string {
+    return JSON.stringify({ params, cactusToken, retryOptions });
+  }
+
   protected constructor(context: LlamaContext) {
     this.context = context
     this.conversationHistoryManager = new ConversationHistoryManager()
@@ -34,64 +40,80 @@ export class CactusLM {
     cactusToken?: string,
     retryOptions?: { maxRetries?: number; delayMs?: number },
   ): Promise<CactusLMReturn> {
+
     if (cactusToken) {
       setCactusToken(cactusToken);
     }
 
-    const maxRetries = retryOptions?.maxRetries ?? 3;
-    const delayMs = retryOptions?.delayMs ?? 1000;
-
-    const configs = [
-      params,
-      { ...params, n_gpu_layers: 0 } 
-    ];
-
-    const sleep = (ms: number): Promise<void> => {
-      return new Promise(resolve => {
-        const start = Date.now();
-        const wait = () => {
-          if (Date.now() - start >= ms) {
-            resolve();
-          } else {
-            Promise.resolve().then(wait);
-          }
-        };
-        wait();
-      });
-    };
+    const key = CactusLM.getCacheKey(params, cactusToken, retryOptions);
+    if (CactusLM._initCache.has(key)) {
+      return CactusLM._initCache.get(key)!;
+    }
 
-    for (const config of configs) {
-      let lastError: Error | null = null;
-      
-      for (let attempt = 1; attempt <= maxRetries; attempt++) {
-        try {
-          const context = await initLlama(config, onProgress);
-          return { lm: new CactusLM(context), error: null };
-        } catch (e) {
-          lastError = e as Error;
-          const isLastConfig = configs.indexOf(config) === configs.length - 1;
-          const isLastAttempt = attempt === maxRetries;
-          
-          Telemetry.error(e as Error, {
-            n_gpu_layers: config.n_gpu_layers ?? null,
-            n_ctx: config.n_ctx ?? null,
-            model: config.model ?? null,
-          });
-          
-          if (!isLastAttempt) {
-            const delay = delayMs * Math.pow(2, attempt - 1);
-            await sleep(delay);
-          } else if (!isLastConfig) {
-            break;
+    const initPromise = (async () => {
+      const maxRetries = retryOptions?.maxRetries ?? 3;
+      const delayMs = retryOptions?.delayMs ?? 1000;
+
+      const configs = [
+        params,
+        { ...params, n_gpu_layers: 0 }
+      ];
+
+      const sleep = (ms: number): Promise<void> => {
+        return new Promise(resolve => {
+          const start = Date.now();
+          const wait = () => {
+            if (Date.now() - start >= ms) {
+              resolve();
+            } else {
+              Promise.resolve().then(wait);
+            }
+          };
+          wait();
+        });
+      };
+
+      for (const config of configs) {
+        let lastError: Error | null = null;
+
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+          try {
+            const context = await initLlama(config, onProgress);
+            return { lm: new CactusLM(context), error: null };
+          } catch (e) {
+            lastError = e as Error;
+            const isLastConfig = configs.indexOf(config) === configs.length - 1;
+            const isLastAttempt = attempt === maxRetries;
+
+            Telemetry.error(e as Error, {
+              n_gpu_layers: config.n_gpu_layers ?? null,
+              n_ctx: config.n_ctx ?? null,
+              model: config.model ?? null,
+            });
+
+            if (!isLastAttempt) {
+              const delay = delayMs * Math.pow(2, attempt - 1);
+              await sleep(delay);
+            } else if (!isLastConfig) {
+              break;
+            }
           }
         }
+
+        if (configs.indexOf(config) === configs.length - 1 && lastError) {
+          return { lm: null, error: lastError };
+        }
       }
-      
-      if (configs.indexOf(config) === configs.length - 1 && lastError) {
-        return { lm: null, error: lastError };
-      }
+      return { lm: null, error: new Error('Failed to initialize CactusLM after all retries') };
+    })();
+
+    CactusLM._initCache.set(key, initPromise);
+
+    const result = await initPromise;
+    if (result.error) {
+      CactusLM._initCache.delete(key); 
     }
-    return { lm: null, error: new Error('Failed to initialize CactusLM after all retries') };
+    return result;
   }
 
   completion = async (
@@ -179,4 +201,9 @@ export class CactusLM {
   async release(): Promise<void> {
     return this.context.release()
   }
+
+  async stopCompletion(): Promise<void> {
+    return await this.context.stopCompletion()
+  }
+
 } 

package/src/telemetry.ts

@@ -1,5 +1,4 @@
 import { Platform } from 'react-native'
-// Import package.json to get version
 const packageJson = require('../package.json');
 import { PROJECT_ID } from './projectId';
 

package/src/tools.ts

@@ -3,7 +3,7 @@ import type { NativeCompletionResult } from "./NativeCactus";
 interface Parameter {
   type: string,
   description: string,
-  required?: boolean // parameter is optional if not specified
+  required?: boolean 
 }
 
 interface Tool {

package/src/vlm.ts

@@ -31,6 +31,12 @@ export class CactusVLM {
   private context: LlamaContext
   protected conversationHistoryManager: ConversationHistoryManager
 
+  private static _initCache: Map<string, Promise<CactusVLMReturn>> = new Map();
+
+  private static getCacheKey(params: VLMContextParams, cactusToken?: string, retryOptions?: { maxRetries?: number; delayMs?: number }): string {
+    return JSON.stringify({ params, cactusToken, retryOptions });
+  }
+
   private constructor(context: LlamaContext) {
     this.context = context
     this.conversationHistoryManager = new ConversationHistoryManager()
@@ -40,34 +46,83 @@ export class CactusVLM {
     params: VLMContextParams,
     onProgress?: (progress: number) => void,
     cactusToken?: string,
+    retryOptions?: { maxRetries?: number; delayMs?: number },
   ): Promise<CactusVLMReturn> {
     if (cactusToken) {
       setCactusToken(cactusToken);
     }
 
-    const configs = [
-      params,
-      { ...params, n_gpu_layers: 0 } 
-    ];
+    const key = CactusVLM.getCacheKey(params, cactusToken, retryOptions);
+    if (CactusVLM._initCache.has(key)) {
+      return CactusVLM._initCache.get(key)!;
+    }
 
-    for (const config of configs) {
-      try {
-        const context = await initLlama(config, onProgress)
-        await initMultimodal(context.id, params.mmproj, false)
-        return {vlm: new CactusVLM(context), error: null}
-      } catch (e) {
-        Telemetry.error(e as Error, {
-          n_gpu_layers: config.n_gpu_layers ?? null,
-          n_ctx: config.n_ctx ?? null,
-          model: config.model ?? null,
+    const initPromise = (async () => {
+      const maxRetries = retryOptions?.maxRetries ?? 3;
+      const delayMs = retryOptions?.delayMs ?? 1000;
+
+      const configs = [
+        params,
+        { ...params, n_gpu_layers: 0 } 
+      ];
+
+      const sleep = (ms: number): Promise<void> => {
+        return new Promise(resolve => {
+          const start = Date.now();
+          const wait = () => {
+            if (Date.now() - start >= ms) {
+              resolve();
+            } else {
+              Promise.resolve().then(wait);
+            }
+          };
+          wait();
         });
-        if (configs.indexOf(config) === configs.length - 1) {
-          return {vlm: null, error: e as Error}
+      };
+
+      for (const config of configs) {
+        let lastError: Error | null = null;
+
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+          try {
+            const context = await initLlama(config, onProgress)
+            await initMultimodal(context.id, params.mmproj, false)
+            return {vlm: new CactusVLM(context), error: null}
+          } catch (e) {
+            lastError = e as Error;
+            const isLastConfig = configs.indexOf(config) === configs.length - 1;
+            const isLastAttempt = attempt === maxRetries;
+
+            Telemetry.error(e as Error, {
+              n_gpu_layers: config.n_gpu_layers ?? null,
+              n_ctx: config.n_ctx ?? null,
+              model: config.model ?? null,
+            });
+
+            if (!isLastAttempt) {
+              const delay = delayMs * Math.pow(2, attempt - 1);
+              await sleep(delay);
+            } else if (!isLastConfig) {
+              break;
+            }
+          }
+        }
+
+        if (configs.indexOf(config) === configs.length - 1 && lastError) {
+          return {vlm: null, error: lastError}
         }
       }
-    }
 
-    return {vlm: null, error: new Error('Failed to initialize CactusVLM')}
+      return {vlm: null, error: new Error('Failed to initialize CactusVLM')}
+    })();
+
+    CactusVLM._initCache.set(key, initPromise);
+
+    const result = await initPromise;
+    if (result.error) {
+      CactusVLM._initCache.delete(key); 
+    }
+    return result;
   }
 
   async completion(
@@ -210,4 +265,8 @@ export class CactusVLM {
   async release(): Promise<void> {
     return this.context.release()
   }
+
+  async stopCompletion(): Promise<void> {
+    return await this.context.stopCompletion()
+  }
 }