cactus-react-native 0.1.4 → 0.2.2

package/src/remote.ts

@@ -0,0 +1,175 @@
+let _cactusToken: string | null = null;
+
+export function setCactusToken(token: string | null): void {
+  _cactusToken = token;
+}
+
+export async function getVertexAIEmbedding(text: string): Promise<number[]> {
+  if (_cactusToken === null) {
+    throw new Error('CactusToken not set. Please call CactusLM.init with cactusToken parameter.');
+  }
+
+  const projectId = 'cactus-v1-452518';
+  const location = 'us-central1';
+  const modelId = 'text-embedding-005';
+  
+  const endpoint = `https://${location}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${location}/publishers/google/models/${modelId}:predict`;
+
+  const headers = {
+    'Authorization': `Bearer ${_cactusToken}`,
+    'Content-Type': 'application/json',
+  };
+
+  const requestBody = {
+    instances: [{ content: text }]
+  };
+
+  const response = await fetch(endpoint, {
+    method: 'POST',
+    headers,
+    body: JSON.stringify(requestBody),
+  });
+
+  if (response.status === 401) {
+    _cactusToken = null;
+    throw new Error('Authentication failed. Please update your cactusToken.');
+  } else if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(`HTTP ${response.status}: ${errorText}`);
+  }
+
+  const responseBody = await response.json();
+  
+  if (responseBody.error) {
+    throw new Error(`API Error: ${responseBody.error.message}`);
+  }
+  
+  const predictions = responseBody.predictions;
+  if (!predictions || predictions.length === 0) {
+    throw new Error('No predictions in response');
+  }
+  
+  const embeddings = predictions[0].embeddings;
+  const values = embeddings.values;
+  
+  return values;
+}
+
+export async function getVertexAICompletion(
+  textPrompt: string,
+  imageData?: string,
+  imagePath?: string,
+  mimeType?: string,
+): Promise<string> {
+  if (_cactusToken === null) {
+    throw new Error('CactusToken not set. Please call CactusVLM.init with cactusToken parameter.');
+  }
+
+  const projectId = 'cactus-v1-452518';
+  const location = 'global';
+  const modelId = 'gemini-2.5-flash-lite-preview-06-17';
+  
+  const endpoint = `https://aiplatform.googleapis.com/v1/projects/${projectId}/locations/${location}/publishers/google/models/${modelId}:generateContent`;
+
+  const headers = {
+    'Authorization': `Bearer ${_cactusToken}`,
+    'Content-Type': 'application/json',
+  };
+
+  const parts: any[] = [];
+  
+  if (imageData) {
+    const detectedMimeType = mimeType || 'image/jpeg';
+    parts.push({
+      inlineData: {
+        mimeType: detectedMimeType,
+        data: imageData
+      }
+    });
+  } else if (imagePath) {
+    const detectedMimeType = mimeType || detectMimeType(imagePath);
+    const RNFS = require('react-native-fs');
+    const base64Data = await RNFS.readFile(imagePath, 'base64');
+    parts.push({
+      inlineData: {
+        mimeType: detectedMimeType,
+        data: base64Data
+      }
+    });
+  }
+  
+  parts.push({ text: textPrompt });
+
+  const requestBody = {
+    contents: {
+      role: 'user',
+      parts: parts,
+    }
+  };
+
+  const response = await fetch(endpoint, {
+    method: 'POST',
+    headers,
+    body: JSON.stringify(requestBody),
+  });
+
+  if (response.status === 401) {
+    _cactusToken = null;
+    throw new Error('Authentication failed. Please update your cactusToken.');
+  } else if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(`HTTP ${response.status}: ${errorText}`);
+  }
+
+  const responseBody = await response.json();
+  
+  if (Array.isArray(responseBody)) {
+    throw new Error('Unexpected response format: received array instead of object');
+  }
+  
+  if (responseBody.error) {
+    throw new Error(`API Error: ${responseBody.error.message}`);
+  }
+  
+  const candidates = responseBody.candidates;
+  if (!candidates || candidates.length === 0) {
+    throw new Error('No candidates in response');
+  }
+  
+  const content = candidates[0].content;
+  const responseParts = content.parts;
+  if (!responseParts || responseParts.length === 0) {
+    throw new Error('No parts in response');
+  }
+  
+  return responseParts[0].text || '';
+}
+
+export async function getTextCompletion(prompt: string): Promise<string> {
+  return getVertexAICompletion(prompt);
+}
+
+export async function getVisionCompletion(prompt: string, imagePath: string): Promise<string> {
+  return getVertexAICompletion(prompt, undefined, imagePath);
+}
+
+export async function getVisionCompletionFromData(prompt: string, imageData: string, mimeType?: string): Promise<string> {
+  return getVertexAICompletion(prompt, imageData, undefined, mimeType);
+}
+
+function detectMimeType(filePath: string): string {
+  const extension = filePath.toLowerCase().split('.').pop();
+  switch (extension) {
+    case 'jpg':
+    case 'jpeg':
+      return 'image/jpeg';
+    case 'png':
+      return 'image/png';
+    case 'gif':
+      return 'image/gif';
+    case 'webp':
+      return 'image/webp';
+    default:
+      return 'image/jpeg';
+  }
+} 

package/src/telemetry.ts

@@ -1,9 +1,19 @@
 import { Platform } from 'react-native'
-import type { ContextParams } from './index';
 // Import package.json to get version
 const packageJson = require('../package.json');
+import { PROJECT_ID } from './projectId';
+
+export interface TelemetryParams {
+  n_gpu_layers: number | null
+  n_ctx: number | null
+  model: string | null
+}
 
 interface TelemetryRecord {
+  project_id: string;
+  device_id?: string;
+  device_manufacturer?: string;
+  device_model?: string;
   os: 'iOS' | 'Android';
   os_version: string;
   framework: string;
@@ -11,9 +21,9 @@ interface TelemetryRecord {
   telemetry_payload?: Record<string, any>;
   error_payload?: Record<string, any>;
   timestamp: string;
-  model_filename: string;
-  n_ctx?: number;
-  n_gpu_layers?: number;
+  model_filename: string | null;
+  n_ctx: number | null;
+  n_gpu_layers: number | null;
 }
 
 interface TelemetryConfig {
@@ -56,37 +66,41 @@ export class Telemetry {
     }
   }
 
-  static track(payload: Record<string, any>, options: ContextParams): void {
+  static track(payload: Record<string, any>, options: TelemetryParams, deviceMetadata?: Record<string, any>): void {
     if (!Telemetry.instance) {
       Telemetry.autoInit();
     }
-    Telemetry.instance!.trackInternal(payload, options);
+    Telemetry.instance!.trackInternal(payload, options, deviceMetadata);
   }
 
-  static error(error: Error, options: ContextParams): void {
+  static error(error: Error, options: TelemetryParams): void {
     if (!Telemetry.instance) {
       Telemetry.autoInit();
     }
     Telemetry.instance!.errorInternal(error, options);
   }
 
-  private trackInternal(payload: Record<string, any>, options: ContextParams): void {
+  private trackInternal(payload: Record<string, any>, options: TelemetryParams, deviceMetadata?: Record<string, any>): void {
     const record: TelemetryRecord = {
+      project_id: PROJECT_ID,
+      device_id: deviceMetadata?.deviceId,
+      device_manufacturer: deviceMetadata?.make,
+      device_model: deviceMetadata?.model,
       os: Platform.OS === 'ios' ? 'iOS' : 'Android',
       os_version: Platform.Version.toString(),
       framework: 'react-native',
       framework_version: packageJson.version,
       telemetry_payload: payload,
       timestamp: new Date().toISOString(),
-      model_filename: Telemetry.getFilename(options.model),
+      model_filename: Telemetry.getFilename(options.model || ''),
       n_ctx: options.n_ctx,
-      n_gpu_layers: options.n_gpu_layers
+      n_gpu_layers: options.n_gpu_layers,
     };
 
     this.sendRecord(record).catch(() => {});
   }
 
-  private errorInternal(error: Error, options: ContextParams): void {
+  private errorInternal(error: Error, options: TelemetryParams): void {
     const errorPayload = {
       message: error.message,
       stack: error.stack,
@@ -94,13 +108,14 @@ export class Telemetry {
     };
 
     const record: TelemetryRecord = {
+      project_id: PROJECT_ID,
       os: Platform.OS === 'ios' ? 'iOS' : 'Android',
       os_version: Platform.Version.toString(),
       framework: 'react-native',
       framework_version: packageJson.version,
       error_payload: errorPayload,
       timestamp: new Date().toISOString(),
-      model_filename: Telemetry.getFilename(options.model),
+      model_filename: Telemetry.getFilename(options.model || ''),
       n_ctx: options.n_ctx,
       n_gpu_layers: options.n_gpu_layers
     };

package/src/vlm.ts

@@ -11,6 +11,7 @@ import type {
   NativeCompletionResult,
 } from './index'
 import { Telemetry } from './telemetry'
+import { setCactusToken, getTextCompletion, getVisionCompletion } from './remote'
 
 interface CactusVLMReturn {
   vlm: CactusVLM | null
@@ -23,21 +24,25 @@ export type VLMContextParams = ContextParams & {
 
 export type VLMCompletionParams = Omit<CompletionParams, 'prompt'> & {
   images?: string[]
+  mode?: string
 }
 
 export class CactusVLM {
   private context: LlamaContext
-  private initParams: VLMContextParams
-
-  private constructor(context: LlamaContext, initParams: VLMContextParams) {
+  
+  private constructor(context: LlamaContext) {
     this.context = context
-    this.initParams = initParams
   }
 
   static async init(
     params: VLMContextParams,
     onProgress?: (progress: number) => void,
+    cactusToken?: string,
   ): Promise<CactusVLMReturn> {
+    if (cactusToken) {
+      setCactusToken(cactusToken);
+    }
+
     const configs = [
       params,
       { ...params, n_gpu_layers: 0 } 
@@ -46,11 +51,14 @@ export class CactusVLM {
     for (const config of configs) {
       try {
         const context = await initLlama(config, onProgress)
-        // Explicitly disable GPU for the multimodal projector for stability.
         await initMultimodal(context.id, params.mmproj, false)
-        return {vlm: new CactusVLM(context, params), error: null}
+        return {vlm: new CactusVLM(context), error: null}
       } catch (e) {
-        Telemetry.error(e as Error, config);
+        Telemetry.error(e as Error, {
+          n_gpu_layers: config.n_gpu_layers ?? null,
+          n_ctx: config.n_ctx ?? null,
+          model: config.model ?? null,
+        });
         if (configs.indexOf(config) === configs.length - 1) {
           return {vlm: null, error: e as Error}
         }
@@ -65,40 +73,111 @@ export class CactusVLM {
     params: VLMCompletionParams = {},
     callback?: (data: any) => void,
   ): Promise<NativeCompletionResult> {
-    const startTime = Date.now();
-    let firstTokenTime: number | null = null;
-    
-    const wrappedCallback = callback ? (data: any) => {
-      if (firstTokenTime === null) firstTokenTime = Date.now();
-      callback(data);
-    } : undefined;
+    const mode = params.mode || 'local';
 
     let result: NativeCompletionResult;
+    let lastError: Error | null = null;
+
+    if (mode === 'remote') {
+      result = await this._handleRemoteCompletion(messages, params, callback);
+    } else if (mode === 'local') {
+      result = await this._handleLocalCompletion(messages, params, callback);
+    } else if (mode === 'localfirst') {
+      try {
+        result = await this._handleLocalCompletion(messages, params, callback);
+      } catch (e) {
+        lastError = e as Error;
+        try {
+          result = await this._handleRemoteCompletion(messages, params, callback);
+        } catch (remoteError) {
+          throw lastError;
+        }
+      }
+    } else if (mode === 'remotefirst') {
+      try {
+        result = await this._handleRemoteCompletion(messages, params, callback);
+      } catch (e) {
+        lastError = e as Error;
+        try {
+          result = await this._handleLocalCompletion(messages, params, callback);
+        } catch (localError) {
+          throw lastError;
+        }
+      }
+    } else {
+      throw new Error('Invalid mode: ' + mode + '. Must be "local", "remote", "localfirst", or "remotefirst"');
+    }
+
+    return result;
+  }
+
+  private async _handleLocalCompletion(
+    messages: CactusOAICompatibleMessage[],
+    params: VLMCompletionParams,
+    callback?: (data: any) => void,
+  ): Promise<NativeCompletionResult> {
     if (params.images && params.images.length > 0) {
       const formattedPrompt = await this.context.getFormattedChat(messages)
       const prompt =
         typeof formattedPrompt === 'string'
           ? formattedPrompt
           : formattedPrompt.prompt
-      result = await multimodalCompletion(
+      return await multimodalCompletion(
         this.context.id,
         prompt,
         params.images,
         { ...params, prompt, emit_partial_completion: !!callback },
       )
     } else {
-      result = await this.context.completion({ messages, ...params }, wrappedCallback)
+      return await this.context.completion({ messages, ...params }, callback)
     }
-    
-    Telemetry.track({
-      event: 'completion',
-      tok_per_sec: (result as any).timings?.predicted_per_second,
-      toks_generated: (result as any).timings?.predicted_n,
-      ttft: firstTokenTime ? firstTokenTime - startTime : null,
-      num_images: params.images?.length,
-    }, this.initParams);
+  }
 
-    return result;
+  private async _handleRemoteCompletion(
+    messages: CactusOAICompatibleMessage[],
+    params: VLMCompletionParams,
+    callback?: (data: any) => void,
+  ): Promise<NativeCompletionResult> {
+    const prompt = messages.map((m) => `${m.role}: ${m.content}`).join('\n');
+    const imagePath = params.images && params.images.length > 0 ? params.images[0] : '';
+    
+    let responseText: string;
+    if (imagePath) {
+      responseText = await getVisionCompletion(prompt, imagePath);
+    } else {
+      responseText = await getTextCompletion(prompt);
+    }
+    
+    if (callback) {
+      for (let i = 0; i < responseText.length; i++) {
+        callback({ token: responseText[i] });
+      }
+    }
+    
+    return {
+      text: responseText,
+      reasoning_content: '',
+      tool_calls: [],
+      content: responseText,
+      tokens_predicted: responseText.split(' ').length,
+      tokens_evaluated: prompt.split(' ').length,
+      truncated: false,
+      stopped_eos: true,
+      stopped_word: '',
+      stopped_limit: 0,
+      stopping_word: '',
+      tokens_cached: 0,
+      timings: {
+        prompt_n: prompt.split(' ').length,
+        prompt_ms: 0,
+        prompt_per_token_ms: 0,
+        prompt_per_second: 0,
+        predicted_n: responseText.split(' ').length,
+        predicted_ms: 0,
+        predicted_per_token_ms: 0,
+        predicted_per_second: 0,
+      },
+    };
   }
 
   async rewind(): Promise<void> {