cactus-react-native 0.1.2 → 0.1.4

package/src/telemetry.ts

@@ -0,0 +1,123 @@
+import { Platform } from 'react-native'
+import type { ContextParams } from './index';
+// Import package.json to get version
+const packageJson = require('../package.json');
+
+interface TelemetryRecord {
+  os: 'iOS' | 'Android';
+  os_version: string;
+  framework: string;
+  framework_version: string;
+  telemetry_payload?: Record<string, any>;
+  error_payload?: Record<string, any>;
+  timestamp: string;
+  model_filename: string;
+  n_ctx?: number;
+  n_gpu_layers?: number;
+}
+
+interface TelemetryConfig {
+  supabaseUrl: string;
+  supabaseKey: string;
+  table?: string;
+}
+
+export class Telemetry {
+  private static instance: Telemetry | null = null;
+  private config: Required<TelemetryConfig>;
+
+  private constructor(config: TelemetryConfig) {
+    this.config = {
+      table: 'telemetry',
+      ...config
+    };
+  }
+
+  private static getFilename(path: string): string {
+    try {
+      return path.split('/').pop() || path.split('\\').pop() || 'unknown';
+    } catch {
+      return 'unknown';
+    }
+  }
+
+  static autoInit(): void {
+    if (!Telemetry.instance) {
+      Telemetry.instance = new Telemetry({
+        supabaseUrl: 'https://vlqqczxwyaodtcdmdmlw.supabase.co',
+        supabaseKey: 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InZscXFjenh3eWFvZHRjZG1kbWx3Iiwicm9sZSI6ImFub24iLCJpYXQiOjE3NTE1MTg2MzIsImV4cCI6MjA2NzA5NDYzMn0.nBzqGuK9j6RZ6mOPWU2boAC_5H9XDs-fPpo5P3WZYbI', // Anon!
+      });
+    }
+  }
+
+  static init(config: TelemetryConfig): void {
+    if (!Telemetry.instance) {
+      Telemetry.instance = new Telemetry(config);
+    }
+  }
+
+  static track(payload: Record<string, any>, options: ContextParams): void {
+    if (!Telemetry.instance) {
+      Telemetry.autoInit();
+    }
+    Telemetry.instance!.trackInternal(payload, options);
+  }
+
+  static error(error: Error, options: ContextParams): void {
+    if (!Telemetry.instance) {
+      Telemetry.autoInit();
+    }
+    Telemetry.instance!.errorInternal(error, options);
+  }
+
+  private trackInternal(payload: Record<string, any>, options: ContextParams): void {
+    const record: TelemetryRecord = {
+      os: Platform.OS === 'ios' ? 'iOS' : 'Android',
+      os_version: Platform.Version.toString(),
+      framework: 'react-native',
+      framework_version: packageJson.version,
+      telemetry_payload: payload,
+      timestamp: new Date().toISOString(),
+      model_filename: Telemetry.getFilename(options.model),
+      n_ctx: options.n_ctx,
+      n_gpu_layers: options.n_gpu_layers
+    };
+
+    this.sendRecord(record).catch(() => {});
+  }
+
+  private errorInternal(error: Error, options: ContextParams): void {
+    const errorPayload = {
+      message: error.message,
+      stack: error.stack,
+      name: error.name,
+    };
+
+    const record: TelemetryRecord = {
+      os: Platform.OS === 'ios' ? 'iOS' : 'Android',
+      os_version: Platform.Version.toString(),
+      framework: 'react-native',
+      framework_version: packageJson.version,
+      error_payload: errorPayload,
+      timestamp: new Date().toISOString(),
+      model_filename: Telemetry.getFilename(options.model),
+      n_ctx: options.n_ctx,
+      n_gpu_layers: options.n_gpu_layers
+    };
+
+    this.sendRecord(record).catch(() => {});
+  }
+
+  private async sendRecord(record: TelemetryRecord): Promise<void> {
+    await (globalThis as any).fetch(`${this.config.supabaseUrl}/rest/v1/${this.config.table}`, {
+      method: 'POST',
+      headers: {
+        'apikey': this.config.supabaseKey,
+        'Authorization': `Bearer ${this.config.supabaseKey}`,
+        'Content-Type': 'application/json',
+        'Prefer': 'return=minimal'
+      },
+      body: JSON.stringify([record])
+    });
+  }
+}

package/src/tools.ts

@@ -1,4 +1,3 @@
-import type { CactusOAICompatibleMessage } from "./chat";
 import type { NativeCompletionResult } from "./NativeCactus";
 
 interface Parameter {
@@ -55,73 +54,33 @@ export class Tools {
   }
 }
 
-export function injectToolsIntoMessages(messages: CactusOAICompatibleMessage[], tools: Tools): CactusOAICompatibleMessage[] {
-  const newMessages = [...messages];
-  const toolsSchemas = tools.getSchemas();
-  const promptToolInjection = `You have access to the following functions. Use them if required - 
-${JSON.stringify(toolsSchemas, null, 2)}
-Only use an available tool if needed. If a tool is chosen, respond ONLY with a JSON object matching the following schema:
-\`\`\`json
-{
-"tool_name": "<name of the tool>",
-"tool_input": {
-"<parameter_name>": "<parameter_value>",
-...
-}
-}
-\`\`\`
-Remember, if you are calling a tool, you must respond with the JSON object and the JSON object ONLY!
-If no tool is needed, respond normally.
-  `;
-  
-  const systemMessage = newMessages.find(m => m.role === 'system');
-  if (!systemMessage) {
-      newMessages.unshift({
-          role: 'system',
-          content: promptToolInjection
-      });
-  } else {
-      systemMessage.content = `${systemMessage.content}\n\n${promptToolInjection}`;
-  }
-  
-  return newMessages;
-}
-
 export async function parseAndExecuteTool(result: NativeCompletionResult, tools: Tools): Promise<{toolCalled: boolean, toolName?: string, toolInput?: any, toolOutput?: any}> {
-  const match = result.content.match(/```json\s*([\s\S]*?)\s*```/);
-  
-  if (!match || !match[1]) return {toolCalled: false};
+  if (!result.tool_calls || result.tool_calls.length === 0) {
+      // console.log('No tool calls found');
+      return {toolCalled: false};
+  }
   
   try {
-      const jsonContent = JSON.parse(match[1]);
-      const { tool_name, tool_input } = jsonContent;
-      // console.log('Calling tool:', tool_name, tool_input);
-      const toolOutput = await tools.execute(tool_name, tool_input) || true;
+      const toolCall = result.tool_calls[0];
+      if (!toolCall) {
+        // console.log('No tool call found');
+        return {toolCalled: false};
+      }
+      const toolName = toolCall.function.name;
+      const toolInput = JSON.parse(toolCall.function.arguments);
+      
+      // console.log('Calling tool:', toolName, toolInput);
+      const toolOutput = await tools.execute(toolName, toolInput);
       // console.log('Tool called result:', toolOutput);
       
       return {
           toolCalled: true,
-          toolName: tool_name,
-          toolInput: tool_input,
+          toolName,
+          toolInput,
           toolOutput
       };
   } catch (error) {
-      // console.error('Error parsing JSON:', match, error);
+      // console.error('Error parsing tool call:', error);
       return {toolCalled: false};
   }
-}
-
-export function updateMessagesWithToolCall(messages: CactusOAICompatibleMessage[], toolName: string, toolInput: any, toolOutput: any): CactusOAICompatibleMessage[] {
-  const newMessages = [...messages];
-
-  newMessages.push({
-      role: 'function-call',
-      content: JSON.stringify({name: toolName, arguments: toolInput}, null, 2)
-  })
-  newMessages.push({
-      role: 'function-response',
-      content: JSON.stringify(toolOutput, null, 2)
-  })
-  
-  return newMessages;
 }

package/src/tts.ts

@@ -0,0 +1,45 @@
+import {
+  LlamaContext,
+  initVocoder,
+  getFormattedAudioCompletion,
+  decodeAudioTokens,
+  releaseVocoder,
+} from './index'
+import type { NativeAudioDecodeResult } from './index'
+
+export class CactusTTS {
+  private context: LlamaContext
+
+  private constructor(context: LlamaContext) {
+    this.context = context
+  }
+
+  static async init(
+    context: LlamaContext,
+    vocoderModelPath: string,
+  ): Promise<CactusTTS> {
+    await initVocoder(context.id, vocoderModelPath)
+    return new CactusTTS(context)
+  }
+
+  async generate(
+    textToSpeak: string,
+    speakerJsonStr: string,
+  ): Promise<NativeAudioDecodeResult> {
+    const { formatted_prompt } = await getFormattedAudioCompletion(
+      this.context.id,
+      speakerJsonStr,
+      textToSpeak,
+    )
+    // This part is simplified. In a real scenario, the tokens from 
+    // the main model would be generated and passed to decodeAudioTokens.
+    // For now, we are assuming a direct path which may not be fully functional
+    // without the main model's token output for TTS.
+    const tokens = (await this.context.tokenize(formatted_prompt)).tokens
+    return decodeAudioTokens(this.context.id, tokens)
+  }
+
+  async release(): Promise<void> {
+    return releaseVocoder(this.context.id)
+  }
+} 

package/src/vlm.ts

@@ -0,0 +1,112 @@
+import {
+  initLlama,
+  initMultimodal,
+  multimodalCompletion,
+  LlamaContext,
+} from './index'
+import type {
+  ContextParams,
+  CompletionParams,
+  CactusOAICompatibleMessage,
+  NativeCompletionResult,
+} from './index'
+import { Telemetry } from './telemetry'
+
+interface CactusVLMReturn {
+  vlm: CactusVLM | null
+  error: Error | null
+}
+
+export type VLMContextParams = ContextParams & {
+  mmproj: string
+}
+
+export type VLMCompletionParams = Omit<CompletionParams, 'prompt'> & {
+  images?: string[]
+}
+
+export class CactusVLM {
+  private context: LlamaContext
+  private initParams: VLMContextParams
+
+  private constructor(context: LlamaContext, initParams: VLMContextParams) {
+    this.context = context
+    this.initParams = initParams
+  }
+
+  static async init(
+    params: VLMContextParams,
+    onProgress?: (progress: number) => void,
+  ): Promise<CactusVLMReturn> {
+    const configs = [
+      params,
+      { ...params, n_gpu_layers: 0 } 
+    ];
+
+    for (const config of configs) {
+      try {
+        const context = await initLlama(config, onProgress)
+        // Explicitly disable GPU for the multimodal projector for stability.
+        await initMultimodal(context.id, params.mmproj, false)
+        return {vlm: new CactusVLM(context, params), error: null}
+      } catch (e) {
+        Telemetry.error(e as Error, config);
+        if (configs.indexOf(config) === configs.length - 1) {
+          return {vlm: null, error: e as Error}
+        }
+      }
+    }
+
+    return {vlm: null, error: new Error('Failed to initialize CactusVLM')}
+  }
+
+  async completion(
+    messages: CactusOAICompatibleMessage[],
+    params: VLMCompletionParams = {},
+    callback?: (data: any) => void,
+  ): Promise<NativeCompletionResult> {
+    const startTime = Date.now();
+    let firstTokenTime: number | null = null;
+    
+    const wrappedCallback = callback ? (data: any) => {
+      if (firstTokenTime === null) firstTokenTime = Date.now();
+      callback(data);
+    } : undefined;
+
+    let result: NativeCompletionResult;
+    if (params.images && params.images.length > 0) {
+      const formattedPrompt = await this.context.getFormattedChat(messages)
+      const prompt =
+        typeof formattedPrompt === 'string'
+          ? formattedPrompt
+          : formattedPrompt.prompt
+      result = await multimodalCompletion(
+        this.context.id,
+        prompt,
+        params.images,
+        { ...params, prompt, emit_partial_completion: !!callback },
+      )
+    } else {
+      result = await this.context.completion({ messages, ...params }, wrappedCallback)
+    }
+    
+    Telemetry.track({
+      event: 'completion',
+      tok_per_sec: (result as any).timings?.predicted_per_second,
+      toks_generated: (result as any).timings?.predicted_n,
+      ttft: firstTokenTime ? firstTokenTime - startTime : null,
+      num_images: params.images?.length,
+    }, this.initParams);
+
+    return result;
+  }
+
+  async rewind(): Promise<void> {
+    // @ts-ignore
+    return this.context?.rewind()
+  }
+
+  async release(): Promise<void> {
+    return this.context.release()
+  }
+}