cactus-react-native 1.0.1 → 1.1.0

package/nitrogen/generated/ios/swift/HybridCactusImageSpec.swift

@@ -0,0 +1,58 @@
+///
+/// HybridCactusImageSpec.swift
+/// This file was generated by nitrogen. DO NOT MODIFY THIS FILE.
+/// https://github.com/mrousavy/nitro
+/// Copyright © 2025 Marc Rousavy @ Margelo
+///
+
+import Foundation
+import NitroModules
+import NitroModules
+
+/// See ``HybridCactusImageSpec``
+public protocol HybridCactusImageSpec_protocol: HybridObject {
+  // Properties
+  
+
+  // Methods
+  func base64(path: String) throws -> Promise<String>
+  func resize(path: String, height: Double, width: Double, quality: Double) throws -> Promise<String>
+}
+
+public extension HybridCactusImageSpec_protocol {
+  /// Default implementation of ``HybridObject.toString``
+  func toString() -> String {
+    return "[HybridObject CactusImage]"
+  }
+}
+
+/// See ``HybridCactusImageSpec``
+open class HybridCactusImageSpec_base {
+  private weak var cxxWrapper: HybridCactusImageSpec_cxx? = nil
+  public init() { }
+  public func getCxxWrapper() -> HybridCactusImageSpec_cxx {
+  #if DEBUG
+    guard self is HybridCactusImageSpec else {
+      fatalError("`self` is not a `HybridCactusImageSpec`! Did you accidentally inherit from `HybridCactusImageSpec_base` instead of `HybridCactusImageSpec`?")
+    }
+  #endif
+    if let cxxWrapper = self.cxxWrapper {
+      return cxxWrapper
+    } else {
+      let cxxWrapper = HybridCactusImageSpec_cxx(self as! HybridCactusImageSpec)
+      self.cxxWrapper = cxxWrapper
+      return cxxWrapper
+    }
+  }
+}
+
+/**
+ * A Swift base-protocol representing the CactusImage HybridObject.
+ * Implement this protocol to create Swift-based instances of CactusImage.
+ * ```swift
+ * class HybridCactusImage : HybridCactusImageSpec {
+ *   // ...
+ * }
+ * ```
+ */
+public typealias HybridCactusImageSpec = HybridCactusImageSpec_protocol & HybridCactusImageSpec_base

package/nitrogen/generated/ios/swift/HybridCactusImageSpec_cxx.swift

@@ -0,0 +1,158 @@
+///
+/// HybridCactusImageSpec_cxx.swift
+/// This file was generated by nitrogen. DO NOT MODIFY THIS FILE.
+/// https://github.com/mrousavy/nitro
+/// Copyright © 2025 Marc Rousavy @ Margelo
+///
+
+import Foundation
+import NitroModules
+import NitroModules
+
+/**
+ * A class implementation that bridges HybridCactusImageSpec over to C++.
+ * In C++, we cannot use Swift protocols - so we need to wrap it in a class to make it strongly defined.
+ *
+ * Also, some Swift types need to be bridged with special handling:
+ * - Enums need to be wrapped in Structs, otherwise they cannot be accessed bi-directionally (Swift bug: https://github.com/swiftlang/swift/issues/75330)
+ * - Other HybridObjects need to be wrapped/unwrapped from the Swift TCxx wrapper
+ * - Throwing methods need to be wrapped with a Result<T, Error> type, as exceptions cannot be propagated to C++
+ */
+open class HybridCactusImageSpec_cxx {
+  /**
+   * The Swift <> C++ bridge's namespace (`margelo::nitro::cactus::bridge::swift`)
+   * from `Cactus-Swift-Cxx-Bridge.hpp`.
+   * This contains specialized C++ templates, and C++ helper functions that can be accessed from Swift.
+   */
+  public typealias bridge = margelo.nitro.cactus.bridge.swift
+
+  /**
+   * Holds an instance of the `HybridCactusImageSpec` Swift protocol.
+   */
+  private var __implementation: any HybridCactusImageSpec
+
+  /**
+   * Holds a weak pointer to the C++ class that wraps the Swift class.
+   */
+  private var __cxxPart: bridge.std__weak_ptr_HybridCactusImageSpec_
+
+  /**
+   * Create a new `HybridCactusImageSpec_cxx` that wraps the given `HybridCactusImageSpec`.
+   * All properties and methods bridge to C++ types.
+   */
+  public init(_ implementation: any HybridCactusImageSpec) {
+    self.__implementation = implementation
+    self.__cxxPart = .init()
+    /* no base class */
+  }
+
+  /**
+   * Get the actual `HybridCactusImageSpec` instance this class wraps.
+   */
+  @inline(__always)
+  public func getHybridCactusImageSpec() -> any HybridCactusImageSpec {
+    return __implementation
+  }
+
+  /**
+   * Casts this instance to a retained unsafe raw pointer.
+   * This acquires one additional strong reference on the object!
+   */
+  public func toUnsafe() -> UnsafeMutableRawPointer {
+    return Unmanaged.passRetained(self).toOpaque()
+  }
+
+  /**
+   * Casts an unsafe pointer to a `HybridCactusImageSpec_cxx`.
+   * The pointer has to be a retained opaque `Unmanaged<HybridCactusImageSpec_cxx>`.
+   * This removes one strong reference from the object!
+   */
+  public class func fromUnsafe(_ pointer: UnsafeMutableRawPointer) -> HybridCactusImageSpec_cxx {
+    return Unmanaged<HybridCactusImageSpec_cxx>.fromOpaque(pointer).takeRetainedValue()
+  }
+
+  /**
+   * Gets (or creates) the C++ part of this Hybrid Object.
+   * The C++ part is a `std::shared_ptr<HybridCactusImageSpec>`.
+   */
+  public func getCxxPart() -> bridge.std__shared_ptr_HybridCactusImageSpec_ {
+    let cachedCxxPart = self.__cxxPart.lock()
+    if Bool(fromCxx: cachedCxxPart) {
+      return cachedCxxPart
+    } else {
+      let newCxxPart = bridge.create_std__shared_ptr_HybridCactusImageSpec_(self.toUnsafe())
+      __cxxPart = bridge.weakify_std__shared_ptr_HybridCactusImageSpec_(newCxxPart)
+      return newCxxPart
+    }
+  }
+
+  
+
+  /**
+   * Get the memory size of the Swift class (plus size of any other allocations)
+   * so the JS VM can properly track it and garbage-collect the JS object if needed.
+   */
+  @inline(__always)
+  public var memorySize: Int {
+    return MemoryHelper.getSizeOf(self.__implementation) + self.__implementation.memorySize
+  }
+
+  /**
+   * Call dispose() on the Swift class.
+   * This _may_ be called manually from JS.
+   */
+  @inline(__always)
+  public func dispose() {
+    self.__implementation.dispose()
+  }
+
+  /**
+   * Call toString() on the Swift class.
+   */
+  @inline(__always)
+  public func toString() -> String {
+    return self.__implementation.toString()
+  }
+
+  // Properties
+  
+
+  // Methods
+  @inline(__always)
+  public final func base64(path: std.string) -> bridge.Result_std__shared_ptr_Promise_std__string___ {
+    do {
+      let __result = try self.__implementation.base64(path: String(path))
+      let __resultCpp = { () -> bridge.std__shared_ptr_Promise_std__string__ in
+        let __promise = bridge.create_std__shared_ptr_Promise_std__string__()
+        let __promiseHolder = bridge.wrap_std__shared_ptr_Promise_std__string__(__promise)
+        __result
+          .then({ __result in __promiseHolder.resolve(std.string(__result)) })
+          .catch({ __error in __promiseHolder.reject(__error.toCpp()) })
+        return __promise
+      }()
+      return bridge.create_Result_std__shared_ptr_Promise_std__string___(__resultCpp)
+    } catch (let __error) {
+      let __exceptionPtr = __error.toCpp()
+      return bridge.create_Result_std__shared_ptr_Promise_std__string___(__exceptionPtr)
+    }
+  }
+  
+  @inline(__always)
+  public final func resize(path: std.string, height: Double, width: Double, quality: Double) -> bridge.Result_std__shared_ptr_Promise_std__string___ {
+    do {
+      let __result = try self.__implementation.resize(path: String(path), height: height, width: width, quality: quality)
+      let __resultCpp = { () -> bridge.std__shared_ptr_Promise_std__string__ in
+        let __promise = bridge.create_std__shared_ptr_Promise_std__string__()
+        let __promiseHolder = bridge.wrap_std__shared_ptr_Promise_std__string__(__promise)
+        __result
+          .then({ __result in __promiseHolder.resolve(std.string(__result)) })
+          .catch({ __error in __promiseHolder.reject(__error.toCpp()) })
+        return __promise
+      }()
+      return bridge.create_Result_std__shared_ptr_Promise_std__string___(__resultCpp)
+    } catch (let __error) {
+      let __exceptionPtr = __error.toCpp()
+      return bridge.create_Result_std__shared_ptr_Promise_std__string___(__exceptionPtr)
+    }
+  }
+}

package/nitrogen/generated/shared/c++/HybridCactusImageSpec.cpp

@@ -0,0 +1,22 @@
+///
+/// HybridCactusImageSpec.cpp
+/// This file was generated by nitrogen. DO NOT MODIFY THIS FILE.
+/// https://github.com/mrousavy/nitro
+/// Copyright © 2025 Marc Rousavy @ Margelo
+///
+
+#include "HybridCactusImageSpec.hpp"
+
+namespace margelo::nitro::cactus {
+
+  void HybridCactusImageSpec::loadHybridMethods() {
+    // load base methods/properties
+    HybridObject::loadHybridMethods();
+    // load custom methods/properties
+    registerHybrids(this, [](Prototype& prototype) {
+      prototype.registerHybridMethod("base64", &HybridCactusImageSpec::base64);
+      prototype.registerHybridMethod("resize", &HybridCactusImageSpec::resize);
+    });
+  }
+
+} // namespace margelo::nitro::cactus

package/nitrogen/generated/shared/c++/HybridCactusImageSpec.hpp

@@ -0,0 +1,64 @@
+///
+/// HybridCactusImageSpec.hpp
+/// This file was generated by nitrogen. DO NOT MODIFY THIS FILE.
+/// https://github.com/mrousavy/nitro
+/// Copyright © 2025 Marc Rousavy @ Margelo
+///
+
+#pragma once
+
+#if __has_include(<NitroModules/HybridObject.hpp>)
+#include <NitroModules/HybridObject.hpp>
+#else
+#error NitroModules cannot be found! Are you sure you installed NitroModules properly?
+#endif
+
+
+
+#include <string>
+#include <NitroModules/Promise.hpp>
+
+namespace margelo::nitro::cactus {
+
+  using namespace margelo::nitro;
+
+  /**
+   * An abstract base class for `CactusImage`
+   * Inherit this class to create instances of `HybridCactusImageSpec` in C++.
+   * You must explicitly call `HybridObject`'s constructor yourself, because it is virtual.
+   * @example
+   * ```cpp
+   * class HybridCactusImage: public HybridCactusImageSpec {
+   * public:
+   *   HybridCactusImage(...): HybridObject(TAG) { ... }
+   *   // ...
+   * };
+   * ```
+   */
+  class HybridCactusImageSpec: public virtual HybridObject {
+    public:
+      // Constructor
+      explicit HybridCactusImageSpec(): HybridObject(TAG) { }
+
+      // Destructor
+      ~HybridCactusImageSpec() override = default;
+
+    public:
+      // Properties
+      
+
+    public:
+      // Methods
+      virtual std::shared_ptr<Promise<std::string>> base64(const std::string& path) = 0;
+      virtual std::shared_ptr<Promise<std::string>> resize(const std::string& path, double height, double width, double quality) = 0;
+
+    protected:
+      // Hybrid Setup
+      void loadHybridMethods() override;
+
+    protected:
+      // Tag for logging
+      static constexpr auto TAG = "CactusImage";
+  };
+
+} // namespace margelo::nitro::cactus

package/nitrogen/generated/shared/c++/HybridCactusSpec.cpp

@@ -16,7 +16,10 @@ namespace margelo::nitro::cactus {
     registerHybrids(this, [](Prototype& prototype) {
       prototype.registerHybridMethod("init", &HybridCactusSpec::init);
       prototype.registerHybridMethod("complete", &HybridCactusSpec::complete);
+      prototype.registerHybridMethod("transcribe", &HybridCactusSpec::transcribe);
       prototype.registerHybridMethod("embed", &HybridCactusSpec::embed);
+      prototype.registerHybridMethod("imageEmbed", &HybridCactusSpec::imageEmbed);
+      prototype.registerHybridMethod("audioEmbed", &HybridCactusSpec::audioEmbed);
       prototype.registerHybridMethod("reset", &HybridCactusSpec::reset);
       prototype.registerHybridMethod("stop", &HybridCactusSpec::stop);
       prototype.registerHybridMethod("destroy", &HybridCactusSpec::destroy);

package/nitrogen/generated/shared/c++/HybridCactusSpec.hpp

@@ -52,9 +52,12 @@ namespace margelo::nitro::cactus {
 
     public:
       // Methods
-      virtual std::shared_ptr<Promise<void>> init(const std::string& modelPath, double contextSize) = 0;
+      virtual std::shared_ptr<Promise<void>> init(const std::string& modelPath, double contextSize, const std::optional<std::string>& corpusDir) = 0;
       virtual std::shared_ptr<Promise<std::string>> complete(const std::string& messagesJson, double responseBufferSize, const std::optional<std::string>& optionsJson, const std::optional<std::string>& toolsJson, const std::optional<std::function<void(const std::string& /* token */, double /* tokenId */)>>& callback) = 0;
+      virtual std::shared_ptr<Promise<std::string>> transcribe(const std::string& audioFilePath, const std::string& prompt, double responseBufferSize, const std::optional<std::string>& optionsJson, const std::optional<std::function<void(const std::string& /* token */, double /* tokenId */)>>& callback) = 0;
       virtual std::shared_ptr<Promise<std::vector<double>>> embed(const std::string& text, double embeddingBufferSize) = 0;
+      virtual std::shared_ptr<Promise<std::vector<double>>> imageEmbed(const std::string& imagePath, double embeddingBufferSize) = 0;
+      virtual std::shared_ptr<Promise<std::vector<double>>> audioEmbed(const std::string& audioPath, double embeddingBufferSize) = 0;
       virtual std::shared_ptr<Promise<void>> reset() = 0;
       virtual std::shared_ptr<Promise<void>> stop() = 0;
       virtual std::shared_ptr<Promise<void>> destroy() = 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cactus-react-native",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "description": "Run AI models locally on mobile devices",
   "main": "./lib/module/index.js",
   "types": "./lib/typescript/src/index.d.ts",

package/src/api/Database.ts

@@ -54,6 +54,33 @@ export class Database {
     return await CactusUtil.registerApp(await response.text());
   }
 
+  public static async getModel(slug: string): Promise<CactusModel> {
+    const response = await fetch(
+      `${this.url}/functions/v1/get-models?slug=${slug}&sdk_name=react&sdk_version=${packageVersion}`,
+      {
+        headers: { apikey: this.key, Authorization: `Bearer ${this.key}` },
+      }
+    );
+
+    if (!response.ok) {
+      throw new Error('Getting model failed');
+    }
+
+    const model = (await response.json()) as CactusModelResponse;
+
+    return {
+      name: model.name,
+      slug: model.slug,
+      quantization: model.quantization,
+      sizeMb: model.size_mb,
+      downloadUrl: model.download_url,
+      supportsToolCalling: model.supports_tool_calling,
+      supportsVision: model.supports_vision,
+      createdAt: model.created_at,
+      isDownloaded: false,
+    };
+  }
+
   public static async getModels(): Promise<CactusModel[]> {
     const response = await fetch(
       `${this.url}/functions/v1/get-models?sdk_name=react&sdk_version=${packageVersion}`,

package/src/api/RemoteLM.ts

@@ -0,0 +1,273 @@
+import { CactusConfig } from '../config/CactusConfig';
+import { CactusImage } from '../native/CactusImage';
+import type {
+  CactusLMCompleteResult,
+  Message,
+  CompleteOptions,
+  Tool,
+} from '../types/CactusLM';
+
+export class RemoteLM {
+  private static readonly completionsUrl =
+    'https://openrouter.ai/api/v1/chat/completions';
+
+  private static readonly defaultModel = 'google/gemini-2.5-flash-lite';
+
+  public static async complete(
+    messages: Message[],
+    options?: CompleteOptions,
+    tools?: { type: 'function'; function: Tool }[],
+    callback?: (token: string) => void
+  ): Promise<CactusLMCompleteResult> {
+    if (!CactusConfig.cactusToken) {
+      throw new Error('cactusToken is required for hybrid completions');
+    }
+
+    const payload = JSON.stringify({
+      model: this.defaultModel,
+      messages: await this.transformMessages(messages),
+      tools,
+      temperature: options?.temperature,
+      top_p: options?.topP,
+      top_k: options?.topK,
+      max_tokens: options?.maxTokens,
+      stop: options?.stopSequences,
+      stream: !!callback,
+    });
+
+    return callback
+      ? await this.streamXHR(payload, callback)
+      : await this.nonStreamFetch(payload);
+  }
+
+  private static getMimeType(filePath: string): string {
+    const extension = filePath.toLowerCase().split('.').pop();
+    switch (extension) {
+      case 'jpg':
+      case 'jpeg':
+        return 'image/jpeg';
+      case 'png':
+        return 'image/png';
+      case 'gif':
+        return 'image/gif';
+      case 'webp':
+        return 'image/webp';
+      default:
+        throw new Error(`Unsupported image format: .${extension}`);
+    }
+  }
+
+  private static async transformMessages(messages: Message[]) {
+    const transformedMessages = [];
+
+    for (const message of messages) {
+      const content: {
+        type: string;
+        text?: string;
+        image_url?: { url: string };
+      }[] = [];
+
+      if (message.content) {
+        content.push({
+          type: 'text',
+          text: message.content,
+        });
+      }
+
+      if (message.images) {
+        for (const image of message.images) {
+          const imagePath = image.replace('file://', '');
+          const mimeType = this.getMimeType(imagePath);
+          const base64Data = await CactusImage.base64(imagePath);
+
+          content.push({
+            type: 'image_url',
+            image_url: {
+              url: `data:${mimeType};base64,${base64Data}`,
+            },
+          });
+        }
+      }
+
+      transformedMessages.push({ role: message.role, content });
+    }
+
+    return transformedMessages;
+  }
+
+  private static streamXHR(
+    payload: string,
+    callback: (token: string) => void
+  ): Promise<CactusLMCompleteResult> {
+    return new Promise((resolve, reject) => {
+      const xhr = new XMLHttpRequest();
+
+      xhr.timeout = 3 * 60 * 1000; // 3 minutes
+      xhr.ontimeout = () =>
+        reject(new Error('Remote streaming completion timed out'));
+
+      xhr.open('POST', this.completionsUrl);
+      xhr.setRequestHeader(
+        'Authorization',
+        `Bearer ${CactusConfig.cactusToken}`
+      );
+      xhr.setRequestHeader('HTTP-Referer', 'https://cactuscompute.com');
+      xhr.setRequestHeader('X-Title', 'Cactus React Native SDK');
+      xhr.setRequestHeader('Content-Type', 'application/json');
+
+      const startTime = performance.now();
+      let lastIndex = 0;
+      let buffer = '';
+      let response = '';
+      let toolCalls: { name: string; arguments: string }[] | undefined;
+      let timeToFirstTokenMs = 0;
+      let prefillTokens = 0;
+      let decodeTokens = 0;
+      let totalTokens = 0;
+
+      xhr.onprogress = () => {
+        const chunk = xhr.responseText.substring(lastIndex);
+        lastIndex = xhr.responseText.length;
+
+        buffer += chunk;
+
+        const lines = buffer.split('\n');
+        buffer = lines.pop() || '';
+
+        for (const line of lines) {
+          if (!line.startsWith('data: ')) {
+            continue;
+          }
+
+          const jsonStr = line.slice(6).trim();
+          if (jsonStr === '[DONE]') {
+            continue;
+          }
+
+          try {
+            const data = JSON.parse(jsonStr);
+
+            if (timeToFirstTokenMs === 0) {
+              timeToFirstTokenMs = performance.now() - startTime;
+            }
+
+            const toolCallChunks = data?.choices?.[0]?.delta?.tool_calls;
+            if (toolCallChunks) {
+              if (!toolCalls) {
+                toolCalls = [];
+              }
+
+              for (const toolCallChunk of toolCallChunks) {
+                const index = toolCallChunk.index;
+
+                if (!toolCalls[index]) {
+                  toolCalls[index] = { name: '', arguments: '' };
+                }
+
+                if (toolCallChunk.function?.name) {
+                  toolCalls[index].name = toolCallChunk.function.name;
+                }
+
+                if (toolCallChunk.function?.arguments) {
+                  toolCalls[index].arguments +=
+                    toolCallChunk.function.arguments;
+                }
+              }
+            }
+
+            const content = data?.choices?.[0]?.delta?.content;
+            if (content) {
+              response += content;
+              callback(content);
+            }
+
+            if (data?.usage) {
+              prefillTokens = data.usage.prompt_tokens;
+              decodeTokens = data.usage.completion_tokens;
+              totalTokens = data.usage.total_tokens;
+            }
+          } catch {}
+        }
+      };
+
+      xhr.onload = () => {
+        const totalTimeMs = performance.now() - startTime;
+        const functionCalls = toolCalls?.map((toolCall) => ({
+          name: toolCall.name,
+          arguments: JSON.parse(toolCall.arguments) as { [key: string]: any },
+        }));
+
+        resolve({
+          success: true,
+          response,
+          functionCalls,
+          timeToFirstTokenMs,
+          totalTimeMs,
+          tokensPerSecond: (decodeTokens * 1000) / totalTimeMs,
+          prefillTokens,
+          decodeTokens,
+          totalTokens,
+        });
+      };
+
+      xhr.onerror = () =>
+        reject(new Error('Remote streaming completion failed'));
+
+      xhr.send(payload);
+    });
+  }
+
+  private static async nonStreamFetch(
+    payload: string
+  ): Promise<CactusLMCompleteResult> {
+    const startTime = performance.now();
+
+    const request = await fetch(this.completionsUrl, {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${CactusConfig.cactusToken}`,
+        'HTTP-Referer': 'https://cactuscompute.com',
+        'X-Title': 'Cactus React Native SDK',
+        'Content-Type': 'application/json',
+      },
+      body: payload,
+    });
+
+    if (!request.ok) {
+      throw new Error('Remote completion failed');
+    }
+
+    const data = await request.json();
+
+    const totalTimeMs = performance.now() - startTime;
+    const decodeTokens = data.usage.completion_tokens;
+
+    const toolCalls:
+      | {
+          function: {
+            name: string;
+            arguments: {
+              [key: string]: any;
+            };
+          };
+        }[]
+      | undefined = data.choices[0].message.tool_calls;
+
+    const functionCalls = toolCalls?.map((toolCall) => ({
+      name: toolCall.function.name,
+      arguments: toolCall.function.arguments,
+    }));
+
+    return {
+      success: true,
+      response: data.choices[0].message.content,
+      functionCalls,
+      timeToFirstTokenMs: totalTimeMs,
+      totalTimeMs,
+      tokensPerSecond: (decodeTokens * 1000) / totalTimeMs,
+      prefillTokens: data.usage.prompt_tokens,
+      decodeTokens,
+      totalTokens: data.usage.total_tokens,
+    };
+  }
+}