cactus-react-native 0.1.3 → 0.2.1

package/src/lm.ts

@@ -7,6 +7,13 @@ import type {
   EmbeddingParams,
   NativeEmbeddingResult,
 } from './index'
+import { Telemetry } from './telemetry'
+import { setCactusToken, getVertexAIEmbedding } from './remote'
+
+interface CactusLMReturn {
+  lm: CactusLM | null
+  error: Error | null
+}
 
 export class CactusLM {
   private context: LlamaContext
@@ -18,9 +25,33 @@ export class CactusLM {
   static async init(
     params: ContextParams,
     onProgress?: (progress: number) => void,
-  ): Promise<CactusLM> {
-    const context = await initLlama(params, onProgress)
-    return new CactusLM(context)
+    cactusToken?: string,
+  ): Promise<CactusLMReturn> {
+    if (cactusToken) {
+      setCactusToken(cactusToken);
+    }
+
+    const configs = [
+      params,
+      { ...params, n_gpu_layers: 0 } 
+    ];
+
+    for (const config of configs) {
+      try {
+        const context = await initLlama(config, onProgress);
+        return { lm: new CactusLM(context), error: null };
+      } catch (e) {
+        Telemetry.error(e as Error, {
+          n_gpu_layers: config.n_gpu_layers ?? null,
+          n_ctx: config.n_ctx ?? null,
+          model: config.model ?? null,
+        });
+        if (configs.indexOf(config) === configs.length - 1) {
+          return { lm: null, error: e as Error };
+        }
+      }
+    }
+    return { lm: null, error: new Error('Failed to initialize CactusLM') };
   }
 
   async completion(
@@ -28,14 +59,58 @@ export class CactusLM {
     params: CompletionParams = {},
     callback?: (data: any) => void,
   ): Promise<NativeCompletionResult> {
-    return this.context.completion({ messages, ...params }, callback)
+    return await this.context.completion({ messages, ...params }, callback);
   }
 
   async embedding(
     text: string,
     params?: EmbeddingParams,
+    mode: string = 'local',
   ): Promise<NativeEmbeddingResult> {
-    return this.context.embedding(text, params)
+    let result: NativeEmbeddingResult;
+    let lastError: Error | null = null;
+
+    if (mode === 'remote') {
+      result = await this._handleRemoteEmbedding(text);
+    } else if (mode === 'local') {
+      result = await this._handleLocalEmbedding(text, params);
+    } else if (mode === 'localfirst') {
+      try {
+        result = await this._handleLocalEmbedding(text, params);
+      } catch (e) {
+        lastError = e as Error;
+        try {
+          result = await this._handleRemoteEmbedding(text);
+        } catch (remoteError) {
+          throw lastError;
+        }
+      }
+    } else if (mode === 'remotefirst') {
+      try {
+        result = await this._handleRemoteEmbedding(text);
+      } catch (e) {
+        lastError = e as Error;
+        try {
+          result = await this._handleLocalEmbedding(text, params);
+        } catch (localError) {
+          throw lastError;
+        }
+      }
+    } else {
+      throw new Error('Invalid mode: ' + mode + '. Must be "local", "remote", "localfirst", or "remotefirst"');
+    }
+    return result;
+  }
+
+  private async _handleLocalEmbedding(text: string, params?: EmbeddingParams): Promise<NativeEmbeddingResult> {
+    return this.context.embedding(text, params);
+  }
+
+  private async _handleRemoteEmbedding(text: string): Promise<NativeEmbeddingResult> {
+    const embeddingValues = await getVertexAIEmbedding(text);
+    return {
+      embedding: embeddingValues,
+    };
   }
 
   async rewind(): Promise<void> {

package/src/projectId.ts

@@ -0,0 +1 @@
+export const PROJECT_ID = 'not-set';

package/src/remote.ts

@@ -0,0 +1,175 @@
+let _cactusToken: string | null = null;
+
+export function setCactusToken(token: string | null): void {
+  _cactusToken = token;
+}
+
+export async function getVertexAIEmbedding(text: string): Promise<number[]> {
+  if (_cactusToken === null) {
+    throw new Error('CactusToken not set. Please call CactusLM.init with cactusToken parameter.');
+  }
+
+  const projectId = 'cactus-v1-452518';
+  const location = 'us-central1';
+  const modelId = 'text-embedding-005';
+  
+  const endpoint = `https://${location}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${location}/publishers/google/models/${modelId}:predict`;
+
+  const headers = {
+    'Authorization': `Bearer ${_cactusToken}`,
+    'Content-Type': 'application/json',
+  };
+
+  const requestBody = {
+    instances: [{ content: text }]
+  };
+
+  const response = await fetch(endpoint, {
+    method: 'POST',
+    headers,
+    body: JSON.stringify(requestBody),
+  });
+
+  if (response.status === 401) {
+    _cactusToken = null;
+    throw new Error('Authentication failed. Please update your cactusToken.');
+  } else if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(`HTTP ${response.status}: ${errorText}`);
+  }
+
+  const responseBody = await response.json();
+  
+  if (responseBody.error) {
+    throw new Error(`API Error: ${responseBody.error.message}`);
+  }
+  
+  const predictions = responseBody.predictions;
+  if (!predictions || predictions.length === 0) {
+    throw new Error('No predictions in response');
+  }
+  
+  const embeddings = predictions[0].embeddings;
+  const values = embeddings.values;
+  
+  return values;
+}
+
+export async function getVertexAICompletion(
+  textPrompt: string,
+  imageData?: string,
+  imagePath?: string,
+  mimeType?: string,
+): Promise<string> {
+  if (_cactusToken === null) {
+    throw new Error('CactusToken not set. Please call CactusVLM.init with cactusToken parameter.');
+  }
+
+  const projectId = 'cactus-v1-452518';
+  const location = 'global';
+  const modelId = 'gemini-2.5-flash-lite-preview-06-17';
+  
+  const endpoint = `https://aiplatform.googleapis.com/v1/projects/${projectId}/locations/${location}/publishers/google/models/${modelId}:generateContent`;
+
+  const headers = {
+    'Authorization': `Bearer ${_cactusToken}`,
+    'Content-Type': 'application/json',
+  };
+
+  const parts: any[] = [];
+  
+  if (imageData) {
+    const detectedMimeType = mimeType || 'image/jpeg';
+    parts.push({
+      inlineData: {
+        mimeType: detectedMimeType,
+        data: imageData
+      }
+    });
+  } else if (imagePath) {
+    const detectedMimeType = mimeType || detectMimeType(imagePath);
+    const RNFS = require('react-native-fs');
+    const base64Data = await RNFS.readFile(imagePath, 'base64');
+    parts.push({
+      inlineData: {
+        mimeType: detectedMimeType,
+        data: base64Data
+      }
+    });
+  }
+  
+  parts.push({ text: textPrompt });
+
+  const requestBody = {
+    contents: {
+      role: 'user',
+      parts: parts,
+    }
+  };
+
+  const response = await fetch(endpoint, {
+    method: 'POST',
+    headers,
+    body: JSON.stringify(requestBody),
+  });
+
+  if (response.status === 401) {
+    _cactusToken = null;
+    throw new Error('Authentication failed. Please update your cactusToken.');
+  } else if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(`HTTP ${response.status}: ${errorText}`);
+  }
+
+  const responseBody = await response.json();
+  
+  if (Array.isArray(responseBody)) {
+    throw new Error('Unexpected response format: received array instead of object');
+  }
+  
+  if (responseBody.error) {
+    throw new Error(`API Error: ${responseBody.error.message}`);
+  }
+  
+  const candidates = responseBody.candidates;
+  if (!candidates || candidates.length === 0) {
+    throw new Error('No candidates in response');
+  }
+  
+  const content = candidates[0].content;
+  const responseParts = content.parts;
+  if (!responseParts || responseParts.length === 0) {
+    throw new Error('No parts in response');
+  }
+  
+  return responseParts[0].text || '';
+}
+
+export async function getTextCompletion(prompt: string): Promise<string> {
+  return getVertexAICompletion(prompt);
+}
+
+export async function getVisionCompletion(prompt: string, imagePath: string): Promise<string> {
+  return getVertexAICompletion(prompt, undefined, imagePath);
+}
+
+export async function getVisionCompletionFromData(prompt: string, imageData: string, mimeType?: string): Promise<string> {
+  return getVertexAICompletion(prompt, imageData, undefined, mimeType);
+}
+
+function detectMimeType(filePath: string): string {
+  const extension = filePath.toLowerCase().split('.').pop();
+  switch (extension) {
+    case 'jpg':
+    case 'jpeg':
+      return 'image/jpeg';
+    case 'png':
+      return 'image/png';
+    case 'gif':
+      return 'image/gif';
+    case 'webp':
+      return 'image/webp';
+    default:
+      return 'image/jpeg';
+  }
+} 

package/src/telemetry.ts

@@ -0,0 +1,138 @@
+import { Platform } from 'react-native'
+// Import package.json to get version
+const packageJson = require('../package.json');
+import { PROJECT_ID } from './projectId';
+
+export interface TelemetryParams {
+  n_gpu_layers: number | null
+  n_ctx: number | null
+  model: string | null
+}
+
+interface TelemetryRecord {
+  project_id: string;
+  device_id?: string;
+  device_manufacturer?: string;
+  device_model?: string;
+  os: 'iOS' | 'Android';
+  os_version: string;
+  framework: string;
+  framework_version: string;
+  telemetry_payload?: Record<string, any>;
+  error_payload?: Record<string, any>;
+  timestamp: string;
+  model_filename: string | null;
+  n_ctx: number | null;
+  n_gpu_layers: number | null;
+}
+
+interface TelemetryConfig {
+  supabaseUrl: string;
+  supabaseKey: string;
+  table?: string;
+}
+
+export class Telemetry {
+  private static instance: Telemetry | null = null;
+  private config: Required<TelemetryConfig>;
+
+  private constructor(config: TelemetryConfig) {
+    this.config = {
+      table: 'telemetry',
+      ...config
+    };
+  }
+
+  private static getFilename(path: string): string {
+    try {
+      return path.split('/').pop() || path.split('\\').pop() || 'unknown';
+    } catch {
+      return 'unknown';
+    }
+  }
+
+  static autoInit(): void {
+    if (!Telemetry.instance) {
+      Telemetry.instance = new Telemetry({
+        supabaseUrl: 'https://vlqqczxwyaodtcdmdmlw.supabase.co',
+        supabaseKey: 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InZscXFjenh3eWFvZHRjZG1kbWx3Iiwicm9sZSI6ImFub24iLCJpYXQiOjE3NTE1MTg2MzIsImV4cCI6MjA2NzA5NDYzMn0.nBzqGuK9j6RZ6mOPWU2boAC_5H9XDs-fPpo5P3WZYbI', // Anon!
+      });
+    }
+  }
+
+  static init(config: TelemetryConfig): void {
+    if (!Telemetry.instance) {
+      Telemetry.instance = new Telemetry(config);
+    }
+  }
+
+  static track(payload: Record<string, any>, options: TelemetryParams, deviceMetadata?: Record<string, any>): void {
+    if (!Telemetry.instance) {
+      Telemetry.autoInit();
+    }
+    Telemetry.instance!.trackInternal(payload, options, deviceMetadata);
+  }
+
+  static error(error: Error, options: TelemetryParams): void {
+    if (!Telemetry.instance) {
+      Telemetry.autoInit();
+    }
+    Telemetry.instance!.errorInternal(error, options);
+  }
+
+  private trackInternal(payload: Record<string, any>, options: TelemetryParams, deviceMetadata?: Record<string, any>): void {
+    const record: TelemetryRecord = {
+      project_id: PROJECT_ID,
+      device_id: deviceMetadata?.deviceId,
+      device_manufacturer: deviceMetadata?.make,
+      device_model: deviceMetadata?.model,
+      os: Platform.OS === 'ios' ? 'iOS' : 'Android',
+      os_version: Platform.Version.toString(),
+      framework: 'react-native',
+      framework_version: packageJson.version,
+      telemetry_payload: payload,
+      timestamp: new Date().toISOString(),
+      model_filename: Telemetry.getFilename(options.model || ''),
+      n_ctx: options.n_ctx,
+      n_gpu_layers: options.n_gpu_layers,
+    };
+
+    this.sendRecord(record).catch(() => {});
+  }
+
+  private errorInternal(error: Error, options: TelemetryParams): void {
+    const errorPayload = {
+      message: error.message,
+      stack: error.stack,
+      name: error.name,
+    };
+
+    const record: TelemetryRecord = {
+      project_id: PROJECT_ID,
+      os: Platform.OS === 'ios' ? 'iOS' : 'Android',
+      os_version: Platform.Version.toString(),
+      framework: 'react-native',
+      framework_version: packageJson.version,
+      error_payload: errorPayload,
+      timestamp: new Date().toISOString(),
+      model_filename: Telemetry.getFilename(options.model || ''),
+      n_ctx: options.n_ctx,
+      n_gpu_layers: options.n_gpu_layers
+    };
+
+    this.sendRecord(record).catch(() => {});
+  }
+
+  private async sendRecord(record: TelemetryRecord): Promise<void> {
+    await (globalThis as any).fetch(`${this.config.supabaseUrl}/rest/v1/${this.config.table}`, {
+      method: 'POST',
+      headers: {
+        'apikey': this.config.supabaseKey,
+        'Authorization': `Bearer ${this.config.supabaseKey}`,
+        'Content-Type': 'application/json',
+        'Prefer': 'return=minimal'
+      },
+      body: JSON.stringify([record])
+    });
+  }
+}

package/src/tools.ts

@@ -1,4 +1,3 @@
-import type { CactusOAICompatibleMessage } from "./chat";
 import type { NativeCompletionResult } from "./NativeCactus";
 
 interface Parameter {
@@ -55,73 +54,33 @@ export class Tools {
   }
 }
 
-export function injectToolsIntoMessages(messages: CactusOAICompatibleMessage[], tools: Tools): CactusOAICompatibleMessage[] {
-  const newMessages = [...messages];
-  const toolsSchemas = tools.getSchemas();
-  const promptToolInjection = `You have access to the following functions. Use them if required - 
-${JSON.stringify(toolsSchemas, null, 2)}
-Only use an available tool if needed. If a tool is chosen, respond ONLY with a JSON object matching the following schema:
-\`\`\`json
-{
-"tool_name": "<name of the tool>",
-"tool_input": {
-"<parameter_name>": "<parameter_value>",
-...
-}
-}
-\`\`\`
-Remember, if you are calling a tool, you must respond with the JSON object and the JSON object ONLY!
-If no tool is needed, respond normally.
-  `;
-  
-  const systemMessage = newMessages.find(m => m.role === 'system');
-  if (!systemMessage) {
-      newMessages.unshift({
-          role: 'system',
-          content: promptToolInjection
-      });
-  } else {
-      systemMessage.content = `${systemMessage.content}\n\n${promptToolInjection}`;
-  }
-  
-  return newMessages;
-}
-
 export async function parseAndExecuteTool(result: NativeCompletionResult, tools: Tools): Promise<{toolCalled: boolean, toolName?: string, toolInput?: any, toolOutput?: any}> {
-  const match = result.content.match(/```json\s*([\s\S]*?)\s*```/);
-  
-  if (!match || !match[1]) return {toolCalled: false};
+  if (!result.tool_calls || result.tool_calls.length === 0) {
+      // console.log('No tool calls found');
+      return {toolCalled: false};
+  }
   
   try {
-      const jsonContent = JSON.parse(match[1]);
-      const { tool_name, tool_input } = jsonContent;
-      // console.log('Calling tool:', tool_name, tool_input);
-      const toolOutput = await tools.execute(tool_name, tool_input) || true;
+      const toolCall = result.tool_calls[0];
+      if (!toolCall) {
+        // console.log('No tool call found');
+        return {toolCalled: false};
+      }
+      const toolName = toolCall.function.name;
+      const toolInput = JSON.parse(toolCall.function.arguments);
+      
+      // console.log('Calling tool:', toolName, toolInput);
+      const toolOutput = await tools.execute(toolName, toolInput);
       // console.log('Tool called result:', toolOutput);
       
       return {
           toolCalled: true,
-          toolName: tool_name,
-          toolInput: tool_input,
+          toolName,
+          toolInput,
           toolOutput
       };
   } catch (error) {
-      // console.error('Error parsing JSON:', match, error);
+      // console.error('Error parsing tool call:', error);
       return {toolCalled: false};
   }
-}
-
-export function updateMessagesWithToolCall(messages: CactusOAICompatibleMessage[], toolName: string, toolInput: any, toolOutput: any): CactusOAICompatibleMessage[] {
-  const newMessages = [...messages];
-
-  newMessages.push({
-      role: 'function-call',
-      content: JSON.stringify({name: toolName, arguments: toolInput}, null, 2)
-  })
-  newMessages.push({
-      role: 'function-response',
-      content: JSON.stringify(toolOutput, null, 2)
-  })
-  
-  return newMessages;
 }

package/src/vlm.ts

@@ -10,6 +10,13 @@ import type {
   CactusOAICompatibleMessage,
   NativeCompletionResult,
 } from './index'
+import { Telemetry } from './telemetry'
+import { setCactusToken, getTextCompletion, getVisionCompletion } from './remote'
+
+interface CactusVLMReturn {
+  vlm: CactusVLM | null
+  error: Error | null
+}
 
 export type VLMContextParams = ContextParams & {
   mmproj: string
@@ -17,11 +24,12 @@ export type VLMContextParams = ContextParams & {
 
 export type VLMCompletionParams = Omit<CompletionParams, 'prompt'> & {
   images?: string[]
+  mode?: string
 }
 
 export class CactusVLM {
   private context: LlamaContext
-
+  
   private constructor(context: LlamaContext) {
     this.context = context
   }
@@ -29,19 +37,84 @@ export class CactusVLM {
   static async init(
     params: VLMContextParams,
     onProgress?: (progress: number) => void,
-  ): Promise<CactusVLM> {
-    const context = await initLlama(params, onProgress)
+    cactusToken?: string,
+  ): Promise<CactusVLMReturn> {
+    if (cactusToken) {
+      setCactusToken(cactusToken);
+    }
+
+    const configs = [
+      params,
+      { ...params, n_gpu_layers: 0 } 
+    ];
 
-    // Explicitly disable GPU for the multimodal projector for stability.
-    await initMultimodal(context.id, params.mmproj, false)
+    for (const config of configs) {
+      try {
+        const context = await initLlama(config, onProgress)
+        await initMultimodal(context.id, params.mmproj, false)
+        return {vlm: new CactusVLM(context), error: null}
+      } catch (e) {
+        Telemetry.error(e as Error, {
+          n_gpu_layers: config.n_gpu_layers ?? null,
+          n_ctx: config.n_ctx ?? null,
+          model: config.model ?? null,
+        });
+        if (configs.indexOf(config) === configs.length - 1) {
+          return {vlm: null, error: e as Error}
+        }
+      }
+    }
 
-    return new CactusVLM(context)
+    return {vlm: null, error: new Error('Failed to initialize CactusVLM')}
   }
 
   async completion(
     messages: CactusOAICompatibleMessage[],
     params: VLMCompletionParams = {},
     callback?: (data: any) => void,
+  ): Promise<NativeCompletionResult> {
+    const mode = params.mode || 'local';
+
+    let result: NativeCompletionResult;
+    let lastError: Error | null = null;
+
+    if (mode === 'remote') {
+      result = await this._handleRemoteCompletion(messages, params, callback);
+    } else if (mode === 'local') {
+      result = await this._handleLocalCompletion(messages, params, callback);
+    } else if (mode === 'localfirst') {
+      try {
+        result = await this._handleLocalCompletion(messages, params, callback);
+      } catch (e) {
+        lastError = e as Error;
+        try {
+          result = await this._handleRemoteCompletion(messages, params, callback);
+        } catch (remoteError) {
+          throw lastError;
+        }
+      }
+    } else if (mode === 'remotefirst') {
+      try {
+        result = await this._handleRemoteCompletion(messages, params, callback);
+      } catch (e) {
+        lastError = e as Error;
+        try {
+          result = await this._handleLocalCompletion(messages, params, callback);
+        } catch (localError) {
+          throw lastError;
+        }
+      }
+    } else {
+      throw new Error('Invalid mode: ' + mode + '. Must be "local", "remote", "localfirst", or "remotefirst"');
+    }
+
+    return result;
+  }
+
+  private async _handleLocalCompletion(
+    messages: CactusOAICompatibleMessage[],
+    params: VLMCompletionParams,
+    callback?: (data: any) => void,
   ): Promise<NativeCompletionResult> {
     if (params.images && params.images.length > 0) {
       const formattedPrompt = await this.context.getFormattedChat(messages)
@@ -49,14 +122,62 @@ export class CactusVLM {
         typeof formattedPrompt === 'string'
           ? formattedPrompt
           : formattedPrompt.prompt
-      return multimodalCompletion(
+      return await multimodalCompletion(
         this.context.id,
         prompt,
         params.images,
         { ...params, prompt, emit_partial_completion: !!callback },
       )
+    } else {
+      return await this.context.completion({ messages, ...params }, callback)
+    }
+  }
+
+  private async _handleRemoteCompletion(
+    messages: CactusOAICompatibleMessage[],
+    params: VLMCompletionParams,
+    callback?: (data: any) => void,
+  ): Promise<NativeCompletionResult> {
+    const prompt = messages.map((m) => `${m.role}: ${m.content}`).join('\n');
+    const imagePath = params.images && params.images.length > 0 ? params.images[0] : '';
+    
+    let responseText: string;
+    if (imagePath) {
+      responseText = await getVisionCompletion(prompt, imagePath);
+    } else {
+      responseText = await getTextCompletion(prompt);
+    }
+    
+    if (callback) {
+      for (let i = 0; i < responseText.length; i++) {
+        callback({ token: responseText[i] });
+      }
     }
-    return this.context.completion({ messages, ...params }, callback)
+    
+    return {
+      text: responseText,
+      reasoning_content: '',
+      tool_calls: [],
+      content: responseText,
+      tokens_predicted: responseText.split(' ').length,
+      tokens_evaluated: prompt.split(' ').length,
+      truncated: false,
+      stopped_eos: true,
+      stopped_word: '',
+      stopped_limit: 0,
+      stopping_word: '',
+      tokens_cached: 0,
+      timings: {
+        prompt_n: prompt.split(' ').length,
+        prompt_ms: 0,
+        prompt_per_token_ms: 0,
+        prompt_per_second: 0,
+        predicted_n: responseText.split(' ').length,
+        predicted_ms: 0,
+        predicted_per_token_ms: 0,
+        predicted_per_second: 0,
+      },
+    };
   }
 
   async rewind(): Promise<void> {