cactus-react-native 1.0.1 → 1.1.0

package/cpp/HybridCactusUtil.cpp

@@ -1,12 +1,11 @@
 #include "HybridCactusUtil.hpp"
 
-namespace margelo::nitro::cactus
-{
+namespace margelo::nitro::cactus {
 
 HybridCactusUtil::HybridCactusUtil() : HybridObject(TAG) {}
 
-std::shared_ptr<Promise<std::string>> HybridCactusUtil::registerApp(const std::string &encryptedData)
-{
+std::shared_ptr<Promise<std::string>>
+HybridCactusUtil::registerApp(const std::string &encryptedData) {
   return Promise<std::string>::async([this, encryptedData]() -> std::string {
     std::lock_guard<std::mutex> lock(this->_mutex);
 
@@ -23,16 +22,19 @@ std::shared_ptr<Promise<std::string>> HybridCactusUtil::registerApp(const std::s
   });
 }
 
-std::shared_ptr<Promise<std::optional<std::string>>> HybridCactusUtil::getDeviceId() {
-  return Promise<std::optional<std::string>>::async([this]() -> std::optional<std::string> {
-    std::lock_guard<std::mutex> lock(this->_mutex);
+std::shared_ptr<Promise<std::optional<std::string>>>
+HybridCactusUtil::getDeviceId() {
+  return Promise<std::optional<std::string>>::async(
+      [this]() -> std::optional<std::string> {
+        std::lock_guard<std::mutex> lock(this->_mutex);
 
-    const char* deviceId = get_device_id();
-    return deviceId ? std::optional<std::string>(deviceId) : std::nullopt;
-  });
+        const char *deviceId = get_device_id();
+        return deviceId ? std::optional<std::string>(deviceId) : std::nullopt;
+      });
 }
 
-std::shared_ptr<Promise<void>> HybridCactusUtil::setAndroidDataDirectory(const std::string &dataDir) {
+std::shared_ptr<Promise<void>>
+HybridCactusUtil::setAndroidDataDirectory(const std::string &dataDir) {
   return Promise<void>::async([this, dataDir]() -> void {
     std::lock_guard<std::mutex> lock(this->_mutex);
 

package/cpp/HybridCactusUtil.hpp

@@ -5,19 +5,19 @@
 
 #include <mutex>
 
-namespace margelo::nitro::cactus
-{
+namespace margelo::nitro::cactus {
 
-class HybridCactusUtil : public HybridCactusUtilSpec
-{
+class HybridCactusUtil : public HybridCactusUtilSpec {
 public:
   HybridCactusUtil();
-  
-  std::shared_ptr<Promise<std::string>> registerApp(const std::string &encryptedData) override;
-  
+
+  std::shared_ptr<Promise<std::string>>
+  registerApp(const std::string &encryptedData) override;
+
   std::shared_ptr<Promise<std::optional<std::string>>> getDeviceId() override;
-  
-  std::shared_ptr<Promise<void>> setAndroidDataDirectory(const std::string &dataDir) override;
+
+  std::shared_ptr<Promise<void>>
+  setAndroidDataDirectory(const std::string &dataDir) override;
 
 private:
   std::mutex _mutex;

package/cpp/cactus_ffi.h

@@ -20,7 +20,7 @@ typedef void* cactus_model_t;
 
 typedef void (*cactus_token_callback)(const char* token, uint32_t token_id, void* user_data);
 
-CACTUS_FFI_EXPORT cactus_model_t cactus_init(const char* model_path, size_t context_size);
+CACTUS_FFI_EXPORT cactus_model_t cactus_init(const char* model_path, size_t context_size, const char* corpus_dir);
 
 CACTUS_FFI_EXPORT int cactus_complete(
     cactus_model_t model,
@@ -33,6 +33,17 @@ CACTUS_FFI_EXPORT int cactus_complete(
     void* user_data
 );
 
+CACTUS_FFI_EXPORT int cactus_transcribe(
+    cactus_model_t model,
+    const char* audio_file_path,
+    const char* prompt,
+    char* response_buffer,
+    size_t buffer_size,
+    const char* options_json,
+    cactus_token_callback callback,
+    void* user_data
+);
+
 
 CACTUS_FFI_EXPORT int cactus_embed(
     cactus_model_t model,
@@ -42,6 +53,22 @@ CACTUS_FFI_EXPORT int cactus_embed(
     size_t* embedding_dim
 );
 
+CACTUS_FFI_EXPORT int cactus_image_embed(
+    cactus_model_t model,
+    const char* image_path,
+    float* embeddings_buffer,
+    size_t buffer_size,
+    size_t* embedding_dim
+);
+
+CACTUS_FFI_EXPORT int cactus_audio_embed(
+    cactus_model_t model,
+    const char* audio_path,
+    float* embeddings_buffer,
+    size_t buffer_size,
+    size_t* embedding_dim
+);
+
 CACTUS_FFI_EXPORT void cactus_reset(cactus_model_t model);
 
 CACTUS_FFI_EXPORT void cactus_stop(cactus_model_t model);

package/ios/HybridCactusImage.swift

@@ -0,0 +1,53 @@
+import Foundation
+import NitroModules
+import UIKit
+
+class HybridCactusImage: HybridCactusImageSpec {
+  func base64(path: String) throws -> Promise<String> {
+    return Promise.async {
+      let fileURL = URL(fileURLWithPath: path)
+      
+      if !FileManager.default.fileExists(atPath: fileURL.path) {
+        throw RuntimeError.error(withMessage: "No such file: \(path)")
+      }
+      
+      let imageData = try Data(contentsOf: fileURL)
+      return imageData.base64EncodedString()
+    }
+  }
+  
+  func resize(path: String, height: Double, width: Double, quality: Double) throws -> Promise<String> {
+    return Promise.async {
+      let fileURL = URL(fileURLWithPath: path)
+      
+      if !FileManager.default.fileExists(atPath: fileURL.path) {
+        throw RuntimeError.error(withMessage: "No such file: \(path)")
+      }
+      
+      guard let imageData = try? Data(contentsOf: fileURL),
+            let image = UIImage(data: imageData) else {
+        throw RuntimeError.error(withMessage: "Failed to load image from: \(path)")
+      }
+      
+      let targetSize = CGSize(width: CGFloat(width), height: CGFloat(height))
+      let renderer = UIGraphicsImageRenderer(size: targetSize)
+      let resizedImage = renderer.image { context in
+        image.draw(in: CGRect(origin: .zero, size: targetSize))
+      }
+      
+      guard let jpegData = resizedImage.jpegData(compressionQuality: CGFloat(quality)) else {
+        throw RuntimeError.error(withMessage: "Failed to compress resized image")
+      }
+      
+      let cacheDir = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0]
+      let fileName = "\(UUID().uuidString).jpg"
+      let outputURL = cacheDir.appendingPathComponent("cactus/images", isDirectory: true)
+        .appendingPathComponent(fileName)
+      
+      try FileManager.default.createDirectory(at: outputURL.deletingLastPathComponent(), withIntermediateDirectories: true)
+      try jpegData.write(to: outputURL)
+      
+      return outputURL.path
+    }
+  }
+}

package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/cactus_ffi.h

@@ -20,7 +20,7 @@ typedef void* cactus_model_t;
 
 typedef void (*cactus_token_callback)(const char* token, uint32_t token_id, void* user_data);
 
-CACTUS_FFI_EXPORT cactus_model_t cactus_init(const char* model_path, size_t context_size);
+CACTUS_FFI_EXPORT cactus_model_t cactus_init(const char* model_path, size_t context_size, const char* corpus_dir);
 
 CACTUS_FFI_EXPORT int cactus_complete(
     cactus_model_t model,
@@ -33,6 +33,17 @@ CACTUS_FFI_EXPORT int cactus_complete(
     void* user_data
 );
 
+CACTUS_FFI_EXPORT int cactus_transcribe(
+    cactus_model_t model,
+    const char* audio_file_path,
+    const char* prompt,
+    char* response_buffer,
+    size_t buffer_size,
+    const char* options_json,
+    cactus_token_callback callback,
+    void* user_data
+);
+
 
 CACTUS_FFI_EXPORT int cactus_embed(
     cactus_model_t model,
@@ -42,6 +53,22 @@ CACTUS_FFI_EXPORT int cactus_embed(
     size_t* embedding_dim
 );
 
+CACTUS_FFI_EXPORT int cactus_image_embed(
+    cactus_model_t model,
+    const char* image_path,
+    float* embeddings_buffer,
+    size_t buffer_size,
+    size_t* embedding_dim
+);
+
+CACTUS_FFI_EXPORT int cactus_audio_embed(
+    cactus_model_t model,
+    const char* audio_path,
+    float* embeddings_buffer,
+    size_t buffer_size,
+    size_t* embedding_dim
+);
+
 CACTUS_FFI_EXPORT void cactus_reset(cactus_model_t model);
 
 CACTUS_FFI_EXPORT void cactus_stop(cactus_model_t model);

package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/engine.h

@@ -8,11 +8,34 @@
 
 #include "../graph/graph.h"
 
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wc99-extensions"
+#pragma clang diagnostic ignored "-Wunused-parameter"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpedantic"
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#endif
+
+extern "C" {
+    #include "../../libs/stb/stb_image.h"
+    #include "../../libs/stb/stb_image_resize2.h"
+}
+
+#ifdef __clang__
+#pragma clang diagnostic pop
+#elif defined(__GNUC__)
+#pragma GCC diagnostic pop
+#endif
+
 class CactusGraph;
 
 namespace cactus {
 namespace engine {
 
+class Siglip2Preprocessor;
+
 struct Config {
     uint32_t vocab_size = 151936;
     uint32_t bos_token_id = 151643;
@@ -31,9 +54,43 @@ struct Config {
     uint32_t moe_every_n_layers = 0;
     bool tie_word_embeddings = true;
 
-    enum class ModelType {QWEN = 0, GEMMA = 1, SMOL = 2, NOMIC = 3, LFM2 = 4};
+    uint32_t vision_hidden_dim = 0;
+    uint32_t vision_num_layers = 0;
+    uint32_t vision_attention_heads = 0;
+    uint32_t vision_image_size = 0;
+    uint32_t vision_patch_size = 0;
+    uint32_t vision_num_channels = 3;
+    uint32_t vision_embed_dim = 0;
+    uint32_t visual_tokens_per_img = 0;
+    bool use_pixel_shuffle = false;
+    uint32_t pixel_shuffle_factor = 1;
+    bool use_image_tokens = false;
+    bool use_layout_tags = false;
+    uint32_t image_seq_len = 64;
+
+    uint32_t global_image_size = 2048;
+    uint32_t max_tile_size = 512;
+    float rescale_factor = 0.00392156862745098f;
+    float image_mean = 0.5f;
+    float image_std = 0.5f;
+    
+    uint32_t downsample_factor = 2;
+    uint32_t min_tiles = 2;
+    uint32_t max_tiles = 10;
+    bool use_thumbnail = true;
+    uint32_t min_image_tokens = 64;
+    uint32_t max_image_tokens = 256;
+        uint32_t max_num_patches = 1024;
+    uint32_t tile_size = 512;
+    float max_pixels_tolerance = 2.0f;
+    bool do_image_splitting = true;
+
+    enum class ModelType {QWEN = 0, GEMMA = 1, SMOL = 2, NOMIC = 3, LFM2 = 5, SIGLIP2 = 6, WHISPER = 7};
     ModelType model_type = ModelType::QWEN;
 
+    enum class ModelVariant {DEFAULT = 0, VLM = 1, EXTRACT = 2, RAG = 3};
+    ModelVariant model_variant = ModelVariant::DEFAULT;
+
     enum class Activation {GELU = 0, SILU = 1};
     Activation activation = Activation::SILU;
 
@@ -70,6 +127,7 @@ struct MergeRule {
 struct ChatMessage {
     std::string role;
     std::string content;
+    std::vector<std::string> images;
 };
 
 class Tokenizer {
@@ -89,18 +147,32 @@ public:
     virtual bool has_chat_template() const { return has_chat_template_; }
 
     virtual bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) = 0;
+    
+    uint32_t get_image_token_id() const { return image_token_id_; }
+    uint32_t get_fake_token_id() const { return fake_token_id_; }
+    uint32_t get_global_img_token_id() const { return global_img_token_id_; }
 
-protected:
 
-    enum class ModelType { UNKNOWN, QWEN, GEMMA, LFM2 , SMOL, BERT };
+    void set_corpus_dir(const std::string& dir) { corpus_dir_ = dir; }
+
+protected:
+    enum class ModelType { UNKNOWN, QWEN, GEMMA, LFM2, SMOL, BERT, WHISPER};
     ModelType model_type_ = ModelType::UNKNOWN;
+    enum class ModelVariant { DEFAULT, VLM, EXTRACT, RAG};
+    ModelVariant model_variant_ = ModelVariant::DEFAULT;
     bool has_chat_template_ = false;
     std::string chat_template_;
+    
+    uint32_t image_token_id_ = 396;
+    uint32_t fake_token_id_ = 49189;
+    uint32_t global_img_token_id_ = 49152;
+    std::string corpus_dir_;
 
     void detect_model_type(const std::string& config_path);
     std::string format_qwen_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt, const std::string& tools_json) const;
     std::string format_gemma_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt, const std::string& tools_json) const;
     std::string format_lfm2_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt, const std::string& tools_json) const;
+    std::string format_lfm2_vl_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt, const std::string& tools_json) const;
     std::string format_smol_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt, const std::string& tools_json) const;
 };
 
@@ -295,28 +367,58 @@ struct KVCache {
 
 class Model {
 public:
+    struct DebugNode {
+        uint32_t layer_idx;
+        std::string name;
+        size_t node_id;
+    };
+
     Model();
     explicit Model(const Config& config);
     virtual ~Model();
 
     const Config& get_config() const { return config_; }
     Tokenizer* get_tokenizer() const { return tokenizer_.get(); }
+    const std::vector<DebugNode>& get_debug_nodes() const;
+
+    virtual bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true);
+    
+    virtual bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size,
+              const std::string& system_prompt = "", bool do_warmup = true);
+
+    virtual uint32_t generate(const std::vector<uint32_t>& tokens, float temperature = -1.0f, float top_p = -1.0f,
+                      size_t top_k = 0, const std::string& profile_file = "");
 
-    bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "");
-    uint32_t generate(const std::vector<uint32_t>& tokens, float temperature = -1.0f, float top_p = -1.0f,
+    virtual uint32_t generate_with_images(const std::vector<uint32_t>& tokens, const std::vector<std::string>& image_paths,
+                                          float temperature = -1.0f, float top_p = -1.0f,
+                                          size_t top_k = 0, const std::string& profile_file = "");
+    
+    virtual uint32_t generate_with_audio(const std::vector<uint32_t>& tokens, const std::vector<float>& mel_bins, float temperature = 0.0f, float top_p = 0.0f,
                       size_t top_k = 0, const std::string& profile_file = "");
 
     std::vector<float> get_embeddings(const std::vector<uint32_t>& tokens, bool pooled = true, const std::string& profile_file = "");
+    
+    virtual std::vector<float> get_image_embeddings(const std::string& image_path);
+    
+    virtual std::vector<float> get_audio_embeddings(const std::vector<float>& mel_bins);
 
     virtual void reset_cache() { kv_cache_.reset(); }
+    
     void set_cache_window(size_t window_size, size_t sink_size = 4) { kv_cache_.set_window_size(window_size, sink_size); }
 
+    void* graph_handle_;
+
 protected:
     virtual size_t forward(const std::vector<uint32_t>& tokens, bool use_cache = false) = 0;
+    
+    virtual size_t forward(const std::vector<float>& mel_bins, const std::vector<uint32_t>& tokens, bool use_cache = false);
+    
     virtual void load_weights_to_graph(CactusGraph* gb) = 0;
+    
     virtual size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx,
                           ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0;
-    virtual size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx,
+    
+                          virtual size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx,
                     ComputeBackend backend) const = 0;
     virtual size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx,
                                   ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0;
@@ -326,7 +428,6 @@ protected:
     Config config_;
     std::unique_ptr<Tokenizer> tokenizer_;
 
-    void* graph_handle_;
     bool initialized_;
     float attention_scale_;
 
@@ -339,9 +440,138 @@ protected:
     size_t embedding_node_id_;
     std::string model_folder_path_;
     size_t output_weight_node_id_;
+
+    mutable std::vector<DebugNode> debug_nodes_;
+
+    void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) const;
+    void clear_debug_nodes();
+
+    bool init_internal(CactusGraph* gb, const std::string& model_folder, size_t context_size,
+                       const std::string& system_prompt, bool do_warmup);
+    bool owns_graph_;
 };
 
 std::unique_ptr<Model> create_model(const std::string& model_folder);
 
+class Siglip2Preprocessor {
+public:
+    struct Config {
+        int patch_size = 16;
+        int downsample_factor = 2;
+        int min_tiles = 2;
+        int max_tiles = 10;
+    bool use_thumbnail = true;
+        int min_image_tokens = 64;
+        int max_image_tokens = 256;
+        int max_num_patches = 1024;
+        int tile_size = 512;
+        float max_pixels_tolerance = 2.0f;
+        bool do_resize = true;
+        bool do_rescale = true;
+        bool do_normalize = true;
+        bool do_convert_rgb = true;
+        bool do_image_splitting = true;
+        float rescale_factor = 1.0f / 255.0f;
+        float image_mean[3] = {0.5f, 0.5f, 0.5f};
+        float image_std[3] = {0.5f, 0.5f, 0.5f};
+    };
+
+    struct PreprocessedImage {
+        std::vector<float> pixel_values;       
+        std::vector<int> pixel_attention_mask; 
+        std::vector<std::pair<int,int>> spatial_shapes;  
+        std::vector<size_t> pixel_values_shape;           
+        std::vector<size_t> pixel_attention_mask_shape;   
+        std::vector<size_t> spatial_shapes_shape;         
+        int num_patches_height;                 
+        int num_patches_width;                  
+        int actual_num_patches;                 
+        int num_tiles;                          
+        int patch_dim;                          
+        int max_patches_per_tile;               
+          
+        int image_rows;                         
+        int image_cols;                         
+        int image_height;                       
+        int image_width;                        
+        int tokens_per_tile;                    
+        int thumbnail_tokens;                   
+        
+        ~PreprocessedImage();
+    };
+
+    struct SpatialShapeResult {
+        std::vector<std::pair<int, int>> shapes;  
+        int grid_rows;                             
+        int grid_cols;                             
+    };
+
+    explicit Siglip2Preprocessor(const Config& config);
+    Siglip2Preprocessor();
+    ~Siglip2Preprocessor();
+
+    PreprocessedImage preprocess_from_file(const std::string& image_path);
+    PreprocessedImage preprocess_from_memory(const unsigned char* img_data, int width, int height, int channels);
+    SpatialShapeResult compute_spatial_shapes(int height, int width);
+
+private:
+    Config config_;
+
+    std::vector<unsigned char> convert_to_rgb(const unsigned char* img_data, int width, int height, int channels);
+    std::pair<int, int> smart_resize(int height, int width);
+    bool is_image_too_large(int height, int width);
+    std::pair<int, int> get_grid_layout(int height, int width);
+    std::pair<int, int> find_closest_aspect_ratio(float aspect_ratio, int width, int height);
+    std::vector<float> resize_image(const unsigned char* img_data, int src_width, int src_height,
+                                    int dst_width, int dst_height, int channels);
+    std::vector<float> normalize_image(const float* img_data, int width, int height, int channels);
+    std::vector<std::vector<float>> convert_image_to_patches(
+        const std::vector<float>& image, int width, int height, int channels, int patch_size);
+    PreprocessedImage pad_patches(const std::vector<std::vector<float>>& tile_patches,
+                                  const std::vector<std::pair<int,int>>& spatial_shapes,
+                                  int patch_dim,
+                                  int max_patches_per_tile);
+    int round_by_factor(int number, int factor);
+};
+
+class AudioProcessor {
+public:
+    struct SpectrogramConfig {
+        size_t n_fft = 400;
+        size_t hop_length = 160;
+        size_t frame_length = 400;
+        float power = 2.0f;
+        bool center = true;
+        const char* pad_mode = "reflect";
+        bool onesided = true;
+        float dither = 0.0f;
+        float mel_floor = 1e-10f;
+        const char* log_mel = nullptr;
+        float reference = 1.0f;
+        float min_value = 1e-10f;
+        bool remove_dc_offset = false;
+    };
+
+    AudioProcessor();
+    ~AudioProcessor();
+
+    void init_mel_filters(size_t num_frequency_bins, size_t num_mel_filters,
+                          float min_freq, float max_freq, size_t sampling_rate);
+
+    std::vector<float> compute_spectrogram(
+        const std::vector<float>& waveform,
+        const SpectrogramConfig& config);
+
+    const std::vector<float>& get_mel_filters() const { return mel_filters_; }
+
+    size_t get_num_mel_filters() const { return num_mel_filters_; }
+    size_t get_num_frequency_bins() const { return num_frequency_bins_; }
+
+private:
+    std::vector<float> mel_filters_;
+    size_t num_frequency_bins_;
+    size_t num_mel_filters_;
+};
+
 }
 }