cactus-react-native 0.1.3 → 0.1.4

package/src/lm.ts

@@ -7,20 +7,43 @@ import type {
   EmbeddingParams,
   NativeEmbeddingResult,
 } from './index'
+import { Telemetry } from './telemetry'
+
+interface CactusLMReturn {
+  lm: CactusLM | null
+  error: Error | null
+}
 
 export class CactusLM {
   private context: LlamaContext
+  private initParams: ContextParams
 
-  private constructor(context: LlamaContext) {
+  private constructor(context: LlamaContext, initParams: ContextParams) {
     this.context = context
+    this.initParams = initParams
   }
 
   static async init(
     params: ContextParams,
     onProgress?: (progress: number) => void,
-  ): Promise<CactusLM> {
-    const context = await initLlama(params, onProgress)
-    return new CactusLM(context)
+  ): Promise<CactusLMReturn> {
+    const configs = [
+      params,
+      { ...params, n_gpu_layers: 0 } 
+    ];
+
+    for (const config of configs) {
+      try {
+        const context = await initLlama(config, onProgress);
+        return { lm: new CactusLM(context, config), error: null };
+      } catch (e) {
+        Telemetry.error(e as Error, config);
+        if (configs.indexOf(config) === configs.length - 1) {
+          return { lm: null, error: e as Error };
+        }
+      }
+    }
+    return { lm: null, error: new Error('Failed to initialize CactusLM') };
   }
 
   async completion(
@@ -28,7 +51,24 @@ export class CactusLM {
     params: CompletionParams = {},
     callback?: (data: any) => void,
   ): Promise<NativeCompletionResult> {
-    return this.context.completion({ messages, ...params }, callback)
+    const startTime = Date.now();
+    let firstTokenTime: number | null = null;
+    
+    const wrappedCallback = callback ? (data: any) => {
+      if (firstTokenTime === null) firstTokenTime = Date.now();
+      callback(data);
+    } : undefined;
+
+    const result = await this.context.completion({ messages, ...params }, wrappedCallback);
+    
+    Telemetry.track({
+      event: 'completion',
+      tok_per_sec: (result as any).timings?.predicted_per_second,
+      toks_generated: (result as any).timings?.predicted_n,
+      ttft: firstTokenTime ? firstTokenTime - startTime : null,
+    }, this.initParams);
+
+    return result;
   }
 
   async embedding(

package/src/telemetry.ts

@@ -0,0 +1,123 @@
+import { Platform } from 'react-native'
+import type { ContextParams } from './index';
+// Import package.json to get version
+const packageJson = require('../package.json');
+
+interface TelemetryRecord {
+  os: 'iOS' | 'Android';
+  os_version: string;
+  framework: string;
+  framework_version: string;
+  telemetry_payload?: Record<string, any>;
+  error_payload?: Record<string, any>;
+  timestamp: string;
+  model_filename: string;
+  n_ctx?: number;
+  n_gpu_layers?: number;
+}
+
+interface TelemetryConfig {
+  supabaseUrl: string;
+  supabaseKey: string;
+  table?: string;
+}
+
+export class Telemetry {
+  private static instance: Telemetry | null = null;
+  private config: Required<TelemetryConfig>;
+
+  private constructor(config: TelemetryConfig) {
+    this.config = {
+      table: 'telemetry',
+      ...config
+    };
+  }
+
+  private static getFilename(path: string): string {
+    try {
+      return path.split('/').pop() || path.split('\\').pop() || 'unknown';
+    } catch {
+      return 'unknown';
+    }
+  }
+
+  static autoInit(): void {
+    if (!Telemetry.instance) {
+      Telemetry.instance = new Telemetry({
+        supabaseUrl: 'https://vlqqczxwyaodtcdmdmlw.supabase.co',
+        supabaseKey: 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InZscXFjenh3eWFvZHRjZG1kbWx3Iiwicm9sZSI6ImFub24iLCJpYXQiOjE3NTE1MTg2MzIsImV4cCI6MjA2NzA5NDYzMn0.nBzqGuK9j6RZ6mOPWU2boAC_5H9XDs-fPpo5P3WZYbI', // Anon!
+      });
+    }
+  }
+
+  static init(config: TelemetryConfig): void {
+    if (!Telemetry.instance) {
+      Telemetry.instance = new Telemetry(config);
+    }
+  }
+
+  static track(payload: Record<string, any>, options: ContextParams): void {
+    if (!Telemetry.instance) {
+      Telemetry.autoInit();
+    }
+    Telemetry.instance!.trackInternal(payload, options);
+  }
+
+  static error(error: Error, options: ContextParams): void {
+    if (!Telemetry.instance) {
+      Telemetry.autoInit();
+    }
+    Telemetry.instance!.errorInternal(error, options);
+  }
+
+  private trackInternal(payload: Record<string, any>, options: ContextParams): void {
+    const record: TelemetryRecord = {
+      os: Platform.OS === 'ios' ? 'iOS' : 'Android',
+      os_version: Platform.Version.toString(),
+      framework: 'react-native',
+      framework_version: packageJson.version,
+      telemetry_payload: payload,
+      timestamp: new Date().toISOString(),
+      model_filename: Telemetry.getFilename(options.model),
+      n_ctx: options.n_ctx,
+      n_gpu_layers: options.n_gpu_layers
+    };
+
+    this.sendRecord(record).catch(() => {});
+  }
+
+  private errorInternal(error: Error, options: ContextParams): void {
+    const errorPayload = {
+      message: error.message,
+      stack: error.stack,
+      name: error.name,
+    };
+
+    const record: TelemetryRecord = {
+      os: Platform.OS === 'ios' ? 'iOS' : 'Android',
+      os_version: Platform.Version.toString(),
+      framework: 'react-native',
+      framework_version: packageJson.version,
+      error_payload: errorPayload,
+      timestamp: new Date().toISOString(),
+      model_filename: Telemetry.getFilename(options.model),
+      n_ctx: options.n_ctx,
+      n_gpu_layers: options.n_gpu_layers
+    };
+
+    this.sendRecord(record).catch(() => {});
+  }
+
+  private async sendRecord(record: TelemetryRecord): Promise<void> {
+    await (globalThis as any).fetch(`${this.config.supabaseUrl}/rest/v1/${this.config.table}`, {
+      method: 'POST',
+      headers: {
+        'apikey': this.config.supabaseKey,
+        'Authorization': `Bearer ${this.config.supabaseKey}`,
+        'Content-Type': 'application/json',
+        'Prefer': 'return=minimal'
+      },
+      body: JSON.stringify([record])
+    });
+  }
+}

package/src/tools.ts

@@ -1,4 +1,3 @@
-import type { CactusOAICompatibleMessage } from "./chat";
 import type { NativeCompletionResult } from "./NativeCactus";
 
 interface Parameter {
@@ -55,73 +54,33 @@ export class Tools {
   }
 }
 
-export function injectToolsIntoMessages(messages: CactusOAICompatibleMessage[], tools: Tools): CactusOAICompatibleMessage[] {
-  const newMessages = [...messages];
-  const toolsSchemas = tools.getSchemas();
-  const promptToolInjection = `You have access to the following functions. Use them if required - 
-${JSON.stringify(toolsSchemas, null, 2)}
-Only use an available tool if needed. If a tool is chosen, respond ONLY with a JSON object matching the following schema:
-\`\`\`json
-{
-"tool_name": "<name of the tool>",
-"tool_input": {
-"<parameter_name>": "<parameter_value>",
-...
-}
-}
-\`\`\`
-Remember, if you are calling a tool, you must respond with the JSON object and the JSON object ONLY!
-If no tool is needed, respond normally.
-  `;
-  
-  const systemMessage = newMessages.find(m => m.role === 'system');
-  if (!systemMessage) {
-      newMessages.unshift({
-          role: 'system',
-          content: promptToolInjection
-      });
-  } else {
-      systemMessage.content = `${systemMessage.content}\n\n${promptToolInjection}`;
-  }
-  
-  return newMessages;
-}
-
 export async function parseAndExecuteTool(result: NativeCompletionResult, tools: Tools): Promise<{toolCalled: boolean, toolName?: string, toolInput?: any, toolOutput?: any}> {
-  const match = result.content.match(/```json\s*([\s\S]*?)\s*```/);
-  
-  if (!match || !match[1]) return {toolCalled: false};
+  if (!result.tool_calls || result.tool_calls.length === 0) {
+      // console.log('No tool calls found');
+      return {toolCalled: false};
+  }
   
   try {
-      const jsonContent = JSON.parse(match[1]);
-      const { tool_name, tool_input } = jsonContent;
-      // console.log('Calling tool:', tool_name, tool_input);
-      const toolOutput = await tools.execute(tool_name, tool_input) || true;
+      const toolCall = result.tool_calls[0];
+      if (!toolCall) {
+        // console.log('No tool call found');
+        return {toolCalled: false};
+      }
+      const toolName = toolCall.function.name;
+      const toolInput = JSON.parse(toolCall.function.arguments);
+      
+      // console.log('Calling tool:', toolName, toolInput);
+      const toolOutput = await tools.execute(toolName, toolInput);
       // console.log('Tool called result:', toolOutput);
       
       return {
           toolCalled: true,
-          toolName: tool_name,
-          toolInput: tool_input,
+          toolName,
+          toolInput,
           toolOutput
       };
   } catch (error) {
-      // console.error('Error parsing JSON:', match, error);
+      // console.error('Error parsing tool call:', error);
       return {toolCalled: false};
   }
-}
-
-export function updateMessagesWithToolCall(messages: CactusOAICompatibleMessage[], toolName: string, toolInput: any, toolOutput: any): CactusOAICompatibleMessage[] {
-  const newMessages = [...messages];
-
-  newMessages.push({
-      role: 'function-call',
-      content: JSON.stringify({name: toolName, arguments: toolInput}, null, 2)
-  })
-  newMessages.push({
-      role: 'function-response',
-      content: JSON.stringify(toolOutput, null, 2)
-  })
-  
-  return newMessages;
 }

package/src/vlm.ts

@@ -10,6 +10,12 @@ import type {
   CactusOAICompatibleMessage,
   NativeCompletionResult,
 } from './index'
+import { Telemetry } from './telemetry'
+
+interface CactusVLMReturn {
+  vlm: CactusVLM | null
+  error: Error | null
+}
 
 export type VLMContextParams = ContextParams & {
   mmproj: string
@@ -21,21 +27,37 @@ export type VLMCompletionParams = Omit<CompletionParams, 'prompt'> & {
 
 export class CactusVLM {
   private context: LlamaContext
+  private initParams: VLMContextParams
 
-  private constructor(context: LlamaContext) {
+  private constructor(context: LlamaContext, initParams: VLMContextParams) {
     this.context = context
+    this.initParams = initParams
   }
 
   static async init(
     params: VLMContextParams,
     onProgress?: (progress: number) => void,
-  ): Promise<CactusVLM> {
-    const context = await initLlama(params, onProgress)
+  ): Promise<CactusVLMReturn> {
+    const configs = [
+      params,
+      { ...params, n_gpu_layers: 0 } 
+    ];
 
-    // Explicitly disable GPU for the multimodal projector for stability.
-    await initMultimodal(context.id, params.mmproj, false)
+    for (const config of configs) {
+      try {
+        const context = await initLlama(config, onProgress)
+        // Explicitly disable GPU for the multimodal projector for stability.
+        await initMultimodal(context.id, params.mmproj, false)
+        return {vlm: new CactusVLM(context, params), error: null}
+      } catch (e) {
+        Telemetry.error(e as Error, config);
+        if (configs.indexOf(config) === configs.length - 1) {
+          return {vlm: null, error: e as Error}
+        }
+      }
+    }
 
-    return new CactusVLM(context)
+    return {vlm: null, error: new Error('Failed to initialize CactusVLM')}
   }
 
   async completion(
@@ -43,20 +65,40 @@ export class CactusVLM {
     params: VLMCompletionParams = {},
     callback?: (data: any) => void,
   ): Promise<NativeCompletionResult> {
+    const startTime = Date.now();
+    let firstTokenTime: number | null = null;
+    
+    const wrappedCallback = callback ? (data: any) => {
+      if (firstTokenTime === null) firstTokenTime = Date.now();
+      callback(data);
+    } : undefined;
+
+    let result: NativeCompletionResult;
     if (params.images && params.images.length > 0) {
       const formattedPrompt = await this.context.getFormattedChat(messages)
       const prompt =
         typeof formattedPrompt === 'string'
           ? formattedPrompt
           : formattedPrompt.prompt
-      return multimodalCompletion(
+      result = await multimodalCompletion(
         this.context.id,
         prompt,
         params.images,
         { ...params, prompt, emit_partial_completion: !!callback },
       )
+    } else {
+      result = await this.context.completion({ messages, ...params }, wrappedCallback)
     }
-    return this.context.completion({ messages, ...params }, callback)
+    
+    Telemetry.track({
+      event: 'completion',
+      tok_per_sec: (result as any).timings?.predicted_per_second,
+      toks_generated: (result as any).timings?.predicted_n,
+      ttft: firstTokenTime ? firstTokenTime - startTime : null,
+      num_images: params.images?.length,
+    }, this.initParams);
+
+    return result;
   }
 
   async rewind(): Promise<void> {