cactus-react-native 1.0.2 → 1.2.0

package/src/api/RemoteLM.ts

@@ -0,0 +1,273 @@
+import { CactusConfig } from '../config/CactusConfig';
+import { CactusImage } from '../native/CactusImage';
+import type {
+  CactusLMCompleteResult,
+  Message,
+  CompleteOptions,
+  Tool,
+} from '../types/CactusLM';
+
+export class RemoteLM {
+  private static readonly completionsUrl =
+    'https://openrouter.ai/api/v1/chat/completions';
+
+  private static readonly defaultModel = 'google/gemini-2.5-flash-lite';
+
+  public static async complete(
+    messages: Message[],
+    options?: CompleteOptions,
+    tools?: { type: 'function'; function: Tool }[],
+    callback?: (token: string) => void
+  ): Promise<CactusLMCompleteResult> {
+    if (!CactusConfig.cactusToken) {
+      throw new Error('cactusToken is required for hybrid completions');
+    }
+
+    const payload = JSON.stringify({
+      model: this.defaultModel,
+      messages: await this.transformMessages(messages),
+      tools,
+      temperature: options?.temperature,
+      top_p: options?.topP,
+      top_k: options?.topK,
+      max_tokens: options?.maxTokens,
+      stop: options?.stopSequences,
+      stream: !!callback,
+    });
+
+    return callback
+      ? await this.streamXHR(payload, callback)
+      : await this.nonStreamFetch(payload);
+  }
+
+  private static getMimeType(filePath: string): string {
+    const extension = filePath.toLowerCase().split('.').pop();
+    switch (extension) {
+      case 'jpg':
+      case 'jpeg':
+        return 'image/jpeg';
+      case 'png':
+        return 'image/png';
+      case 'gif':
+        return 'image/gif';
+      case 'webp':
+        return 'image/webp';
+      default:
+        throw new Error(`Unsupported image format: .${extension}`);
+    }
+  }
+
+  private static async transformMessages(messages: Message[]) {
+    const transformedMessages = [];
+
+    for (const message of messages) {
+      const content: {
+        type: string;
+        text?: string;
+        image_url?: { url: string };
+      }[] = [];
+
+      if (message.content) {
+        content.push({
+          type: 'text',
+          text: message.content,
+        });
+      }
+
+      if (message.images) {
+        for (const image of message.images) {
+          const imagePath = image.replace('file://', '');
+          const mimeType = this.getMimeType(imagePath);
+          const base64Data = await CactusImage.base64(imagePath);
+
+          content.push({
+            type: 'image_url',
+            image_url: {
+              url: `data:${mimeType};base64,${base64Data}`,
+            },
+          });
+        }
+      }
+
+      transformedMessages.push({ role: message.role, content });
+    }
+
+    return transformedMessages;
+  }
+
+  private static streamXHR(
+    payload: string,
+    callback: (token: string) => void
+  ): Promise<CactusLMCompleteResult> {
+    return new Promise((resolve, reject) => {
+      const xhr = new XMLHttpRequest();
+
+      xhr.timeout = 3 * 60 * 1000; // 3 minutes
+      xhr.ontimeout = () =>
+        reject(new Error('Remote streaming completion timed out'));
+
+      xhr.open('POST', this.completionsUrl);
+      xhr.setRequestHeader(
+        'Authorization',
+        `Bearer ${CactusConfig.cactusToken}`
+      );
+      xhr.setRequestHeader('HTTP-Referer', 'https://cactuscompute.com');
+      xhr.setRequestHeader('X-Title', 'Cactus React Native SDK');
+      xhr.setRequestHeader('Content-Type', 'application/json');
+
+      const startTime = performance.now();
+      let lastIndex = 0;
+      let buffer = '';
+      let response = '';
+      let toolCalls: { name: string; arguments: string }[] | undefined;
+      let timeToFirstTokenMs = 0;
+      let prefillTokens = 0;
+      let decodeTokens = 0;
+      let totalTokens = 0;
+
+      xhr.onprogress = () => {
+        const chunk = xhr.responseText.substring(lastIndex);
+        lastIndex = xhr.responseText.length;
+
+        buffer += chunk;
+
+        const lines = buffer.split('\n');
+        buffer = lines.pop() || '';
+
+        for (const line of lines) {
+          if (!line.startsWith('data: ')) {
+            continue;
+          }
+
+          const jsonStr = line.slice(6).trim();
+          if (jsonStr === '[DONE]') {
+            continue;
+          }
+
+          try {
+            const data = JSON.parse(jsonStr);
+
+            if (timeToFirstTokenMs === 0) {
+              timeToFirstTokenMs = performance.now() - startTime;
+            }
+
+            const toolCallChunks = data?.choices?.[0]?.delta?.tool_calls;
+            if (toolCallChunks) {
+              if (!toolCalls) {
+                toolCalls = [];
+              }
+
+              for (const toolCallChunk of toolCallChunks) {
+                const index = toolCallChunk.index;
+
+                if (!toolCalls[index]) {
+                  toolCalls[index] = { name: '', arguments: '' };
+                }
+
+                if (toolCallChunk.function?.name) {
+                  toolCalls[index].name = toolCallChunk.function.name;
+                }
+
+                if (toolCallChunk.function?.arguments) {
+                  toolCalls[index].arguments +=
+                    toolCallChunk.function.arguments;
+                }
+              }
+            }
+
+            const content = data?.choices?.[0]?.delta?.content;
+            if (content) {
+              response += content;
+              callback(content);
+            }
+
+            if (data?.usage) {
+              prefillTokens = data.usage.prompt_tokens;
+              decodeTokens = data.usage.completion_tokens;
+              totalTokens = data.usage.total_tokens;
+            }
+          } catch {}
+        }
+      };
+
+      xhr.onload = () => {
+        const totalTimeMs = performance.now() - startTime;
+        const functionCalls = toolCalls?.map((toolCall) => ({
+          name: toolCall.name,
+          arguments: JSON.parse(toolCall.arguments) as { [key: string]: any },
+        }));
+
+        resolve({
+          success: true,
+          response,
+          functionCalls,
+          timeToFirstTokenMs,
+          totalTimeMs,
+          tokensPerSecond: (decodeTokens * 1000) / totalTimeMs,
+          prefillTokens,
+          decodeTokens,
+          totalTokens,
+        });
+      };
+
+      xhr.onerror = () =>
+        reject(new Error('Remote streaming completion failed'));
+
+      xhr.send(payload);
+    });
+  }
+
+  private static async nonStreamFetch(
+    payload: string
+  ): Promise<CactusLMCompleteResult> {
+    const startTime = performance.now();
+
+    const request = await fetch(this.completionsUrl, {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${CactusConfig.cactusToken}`,
+        'HTTP-Referer': 'https://cactuscompute.com',
+        'X-Title': 'Cactus React Native SDK',
+        'Content-Type': 'application/json',
+      },
+      body: payload,
+    });
+
+    if (!request.ok) {
+      throw new Error('Remote completion failed');
+    }
+
+    const data = await request.json();
+
+    const totalTimeMs = performance.now() - startTime;
+    const decodeTokens = data.usage.completion_tokens;
+
+    const toolCalls:
+      | {
+          function: {
+            name: string;
+            arguments: {
+              [key: string]: any;
+            };
+          };
+        }[]
+      | undefined = data.choices[0].message.tool_calls;
+
+    const functionCalls = toolCalls?.map((toolCall) => ({
+      name: toolCall.function.name,
+      arguments: toolCall.function.arguments,
+    }));
+
+    return {
+      success: true,
+      response: data.choices[0].message.content,
+      functionCalls,
+      timeToFirstTokenMs: totalTimeMs,
+      totalTimeMs,
+      tokensPerSecond: (decodeTokens * 1000) / totalTimeMs,
+      prefillTokens: data.usage.prompt_tokens,
+      decodeTokens,
+      totalTokens: data.usage.total_tokens,
+    };
+  }
+}

package/src/classes/CactusLM.ts

@@ -5,7 +5,8 @@ import type {
   CactusLMCompleteResult,
   CactusLMEmbedParams,
   CactusLMEmbedResult,
-  CactusLMGetModelsParams,
+  CactusLMImageEmbedParams,
+  CactusLMImageEmbedResult,
   CactusLMParams,
 } from '../types/CactusLM';
 import type { CactusModel } from '../types/CactusModel';
@@ -13,6 +14,7 @@ import { Telemetry } from '../telemetry/Telemetry';
 import { CactusConfig } from '../config/CactusConfig';
 import { Database } from '../api/Database';
 import { getErrorMessage } from '../utils/error';
+import { RemoteLM } from '../api/RemoteLM';
 
 export class CactusLM {
   private readonly cactus = new Cactus();
@@ -30,11 +32,14 @@ export class CactusLM {
   private static readonly defaultCompleteOptions = {
     maxTokens: 512,
   };
+  private static readonly defaultCompleteMode = 'local';
   private static readonly defaultEmbedBufferSize = 2048;
 
-  private static readonly modelsInfoPath = 'models/info.json';
+  private static cactusModelsCache: CactusModel[] | null = null;
 
   constructor({ model, contextSize, corpusDir }: CactusLMParams = {}) {
+    Telemetry.init(CactusConfig.telemetryToken);
+
     this.model = model ?? CactusLM.defaultModel;
     this.contextSize = contextSize ?? CactusLM.defaultContextSize;
     this.corpusDir = corpusDir;
@@ -54,8 +59,12 @@ export class CactusLM {
 
     this.isDownloading = true;
     try {
-      await CactusFileSystem.downloadModel(this.model, onProgress);
-      await this.getModels({ forceRefresh: true });
+      const model = await Database.getModel(this.model);
+      await CactusFileSystem.downloadModel(
+        this.model,
+        model.downloadUrl,
+        onProgress
+      );
     } finally {
       this.isDownloading = false;
     }
@@ -66,10 +75,6 @@ export class CactusLM {
       return;
     }
 
-    if (!Telemetry.isInitialized()) {
-      await Telemetry.init(CactusConfig.telemetryToken);
-    }
-
     if (!(await CactusFileSystem.modelExists(this.model))) {
       throw new Error(`Model "${this.model}" is not downloaded`);
     }
@@ -91,25 +96,32 @@ export class CactusLM {
     options,
     tools,
     onToken,
+    mode,
   }: CactusLMCompleteParams): Promise<CactusLMCompleteResult> {
     if (this.isGenerating) {
       throw new Error('CactusLM is already generating');
     }
 
-    await this.init();
-
     options = { ...CactusLM.defaultCompleteOptions, ...options };
+    const toolsInternal = tools?.map((tool) => ({
+      type: 'function' as const,
+      function: tool,
+    }));
+    mode = mode ?? CactusLM.defaultCompleteMode;
+
     const responseBufferSize =
       8 * (options.maxTokens ?? CactusLM.defaultCompleteOptions.maxTokens) +
       256;
 
-    this.isGenerating = true;
     try {
+      await this.init();
+
+      this.isGenerating = true;
       const result = await this.cactus.complete(
         messages,
         responseBufferSize,
         options,
-        tools,
+        toolsInternal,
         onToken
       );
       Telemetry.logCompletion(
@@ -119,9 +131,25 @@ export class CactusLM {
         result
       );
       return result;
-    } catch (error) {
-      Telemetry.logCompletion(this.model, false, getErrorMessage(error));
-      throw error;
+    } catch (localError) {
+      if (mode === 'local') {
+        Telemetry.logCompletion(this.model, false, getErrorMessage(localError));
+        throw localError;
+      }
+
+      Telemetry.logCompletion(
+        this.model,
+        false,
+        `Local completion error: ${getErrorMessage(localError)}. Falling back to remote completion.`
+      );
+
+      try {
+        return RemoteLM.complete(messages, options, toolsInternal, onToken);
+      } catch (remoteError) {
+        throw new Error(
+          `Remote completion error: ${getErrorMessage(remoteError)}`
+        );
+      }
     } finally {
       this.isGenerating = false;
     }
@@ -152,6 +180,31 @@ export class CactusLM {
     }
   }
 
+  public async imageEmbed({
+    imagePath,
+  }: CactusLMImageEmbedParams): Promise<CactusLMImageEmbedResult> {
+    if (this.isGenerating) {
+      throw new Error('CactusLM is already generating');
+    }
+
+    await this.init();
+
+    this.isGenerating = true;
+    try {
+      const embedding = await this.cactus.imageEmbed(
+        imagePath,
+        CactusLM.defaultEmbedBufferSize
+      );
+      Telemetry.logImageEmbedding(this.model, true);
+      return { embedding };
+    } catch (error) {
+      Telemetry.logImageEmbedding(this.model, false, getErrorMessage(error));
+      throw error;
+    } finally {
+      this.isGenerating = false;
+    }
+  }
+
   public stop(): Promise<void> {
     return this.cactus.stop();
   }
@@ -172,34 +225,15 @@ export class CactusLM {
     this.isInitialized = false;
   }
 
-  public async getModels({
-    forceRefresh = false,
-  }: CactusLMGetModelsParams = {}): Promise<CactusModel[]> {
-    if (
-      !forceRefresh &&
-      (await CactusFileSystem.fileExists(CactusLM.modelsInfoPath))
-    ) {
-      try {
-        return JSON.parse(
-          await CactusFileSystem.readFile(CactusLM.modelsInfoPath)
-        );
-      } catch {
-        // Delete corrupted models info
-        await CactusFileSystem.deleteFile(CactusLM.modelsInfoPath);
-      }
+  public async getModels(): Promise<CactusModel[]> {
+    if (CactusLM.cactusModelsCache) {
+      return CactusLM.cactusModelsCache;
     }
-
     const models = await Database.getModels();
-
     for (const model of models) {
       model.isDownloaded = await CactusFileSystem.modelExists(model.slug);
     }
-
-    await CactusFileSystem.writeFile(
-      CactusLM.modelsInfoPath,
-      JSON.stringify(models)
-    );
-
+    CactusLM.cactusModelsCache = models;
     return models;
   }
 }

package/src/classes/CactusSTT.ts

@@ -0,0 +1,188 @@
+import { Cactus, CactusFileSystem } from '../native';
+import type {
+  CactusSTTDownloadParams,
+  CactusSTTTranscribeParams,
+  CactusSTTTranscribeResult,
+  CactusSTTParams,
+  CactusSTTAudioEmbedParams,
+  CactusSTTAudioEmbedResult,
+} from '../types/CactusSTT';
+import type { CactusModel } from '../types/CactusModel';
+import { Telemetry } from '../telemetry/Telemetry';
+import { CactusConfig } from '../config/CactusConfig';
+import { Database } from '../api/Database';
+import { getErrorMessage } from '../utils/error';
+
+export class CactusSTT {
+  private readonly cactus = new Cactus();
+
+  private readonly model: string;
+  private readonly contextSize: number;
+
+  private isDownloading = false;
+  private isInitialized = false;
+  private isGenerating = false;
+
+  private static readonly defaultModel = 'whisper-small';
+  private static readonly defaultContextSize = 2048;
+  private static readonly defaultPrompt =
+    '<|startoftranscript|><|en|><|transcribe|><|notimestamps|>';
+  private static readonly defaultTranscribeOptions = {
+    maxTokens: 512,
+  };
+  private static readonly defaultEmbedBufferSize = 4096;
+
+  private static cactusModelsCache: CactusModel[] | null = null;
+
+  constructor({ model, contextSize }: CactusSTTParams = {}) {
+    Telemetry.init(CactusConfig.telemetryToken);
+
+    this.model = model ?? CactusSTT.defaultModel;
+    this.contextSize = contextSize ?? CactusSTT.defaultContextSize;
+  }
+
+  public async download({
+    onProgress,
+  }: CactusSTTDownloadParams = {}): Promise<void> {
+    if (this.isDownloading) {
+      throw new Error('CactusSTT is already downloading');
+    }
+
+    if (await CactusFileSystem.modelExists(this.model)) {
+      onProgress?.(1.0);
+      return;
+    }
+
+    this.isDownloading = true;
+    try {
+      await CactusFileSystem.downloadModel(
+        this.model,
+        `https://vlqqczxwyaodtcdmdmlw.supabase.co/storage/v1/object/public/voice-models/${this.model}.zip`,
+        onProgress
+      );
+    } finally {
+      this.isDownloading = false;
+    }
+  }
+
+  public async init(): Promise<void> {
+    if (this.isInitialized) {
+      return;
+    }
+
+    if (!(await CactusFileSystem.modelExists(this.model))) {
+      throw new Error(`Model "${this.model}" is not downloaded`);
+    }
+
+    const modelPath = await CactusFileSystem.getModelPath(this.model);
+
+    try {
+      await this.cactus.init(modelPath, this.contextSize);
+      Telemetry.logInit(this.model, true);
+      this.isInitialized = true;
+    } catch (error) {
+      Telemetry.logInit(this.model, false, getErrorMessage(error));
+      throw error;
+    }
+  }
+
+  public async transcribe({
+    audioFilePath,
+    prompt,
+    options,
+    onToken,
+  }: CactusSTTTranscribeParams): Promise<CactusSTTTranscribeResult> {
+    if (this.isGenerating) {
+      throw new Error('CactusSTT is already generating');
+    }
+
+    await this.init();
+
+    prompt = prompt ?? CactusSTT.defaultPrompt;
+    options = { ...CactusSTT.defaultTranscribeOptions, ...options };
+
+    const responseBufferSize =
+      8 * (options.maxTokens ?? CactusSTT.defaultTranscribeOptions.maxTokens) +
+      256;
+
+    this.isGenerating = true;
+    try {
+      const result = await this.cactus.transcribe(
+        audioFilePath,
+        prompt,
+        responseBufferSize,
+        options,
+        onToken
+      );
+      Telemetry.logTranscribe(
+        this.model,
+        result.success,
+        result.success ? undefined : result.response,
+        result
+      );
+      return result;
+    } catch (error) {
+      Telemetry.logTranscribe(this.model, false, getErrorMessage(error));
+      throw error;
+    } finally {
+      this.isGenerating = false;
+    }
+  }
+
+  public async audioEmbed({
+    audioPath,
+  }: CactusSTTAudioEmbedParams): Promise<CactusSTTAudioEmbedResult> {
+    if (this.isGenerating) {
+      throw new Error('CactusSTT is already generating');
+    }
+
+    await this.init();
+
+    this.isGenerating = true;
+    try {
+      const embedding = await this.cactus.audioEmbed(
+        audioPath,
+        CactusSTT.defaultEmbedBufferSize
+      );
+      Telemetry.logAudioEmbedding(this.model, true);
+      return { embedding };
+    } catch (error) {
+      Telemetry.logAudioEmbedding(this.model, false, getErrorMessage(error));
+      throw error;
+    } finally {
+      this.isGenerating = false;
+    }
+  }
+
+  public stop(): Promise<void> {
+    return this.cactus.stop();
+  }
+
+  public async reset(): Promise<void> {
+    await this.stop();
+    return this.cactus.reset();
+  }
+
+  public async destroy(): Promise<void> {
+    if (!this.isInitialized) {
+      return;
+    }
+
+    await this.stop();
+    await this.cactus.destroy();
+
+    this.isInitialized = false;
+  }
+
+  public async getModels(): Promise<CactusModel[]> {
+    if (CactusSTT.cactusModelsCache) {
+      return CactusSTT.cactusModelsCache;
+    }
+    const models = await Database.getModels();
+    for (const model of models) {
+      model.isDownloaded = await CactusFileSystem.modelExists(model.slug);
+    }
+    CactusSTT.cactusModelsCache = models;
+    return models;
+  }
+}

package/src/config/CactusConfig.ts

@@ -1,4 +1,8 @@
 export class CactusConfig {
+  // Telemetry
   public static telemetryToken?: string;
   public static isTelemetryEnabled: boolean = true;
+
+  // Hybrid mode
+  public static cactusToken?: string;
 }

package/src/constants/packageVersion.ts

@@ -1 +1 @@
-export const packageVersion = '1.0.2';
+export const packageVersion = '1.2.0';