cactus-react-native 1.2.1 → 1.5.0

package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/{ffi_utils.h → cactus_utils.h}

@@ -1,5 +1,5 @@
-#ifndef CACTUS_FFI_UTILS_H
-#define CACTUS_FFI_UTILS_H
+#ifndef CACTUS_UTILS_H
+#define CACTUS_UTILS_H
 
 #include "../engine/engine.h"
 #include <string>
@@ -12,16 +12,65 @@
 #include <iostream>
 #include <filesystem>
 #include <cctype>
+#include <memory>
+#include <atomic>
+#include <mutex>
+#include <random>
+
+#ifdef __APPLE__
+#include <uuid/uuid.h>
+#endif
+
+struct CactusModelHandle {
+    std::unique_ptr<cactus::engine::Model> model;
+    std::atomic<bool> should_stop;
+    std::vector<uint32_t> processed_tokens;
+    std::mutex model_mutex;
+    std::string model_name;
+
+    CactusModelHandle() : should_stop(false) {}
+};
+
+extern std::string last_error_message;
+
+bool matches_stop_sequence(const std::vector<uint32_t>& generated_tokens,
+                           const std::vector<std::vector<uint32_t>>& stop_sequences);
 
 namespace cactus {
 namespace ffi {
 
+#ifndef CACTUS_VERSION
+#define CACTUS_VERSION "unknown"
+#endif
+
+inline const char* getVersion() {
+    return CACTUS_VERSION;
+}
+
+inline std::string generateUUID() {
+#ifdef __APPLE__
+    uuid_t uuid;
+    uuid_generate_random(uuid);
+    char uuid_str[37];
+    uuid_unparse_lower(uuid, uuid_str);
+    return std::string(uuid_str);
+#endif
+}
+
 struct ToolFunction {
     std::string name;
     std::string description;
     std::unordered_map<std::string, std::string> parameters;
 };
 
+} // namespace ffi
+} // namespace cactus
+
+#include "gemma_tools.h"
+
+namespace cactus {
+namespace ffi {
+
 inline void handle_error_response(const std::string& error_message, char* response_buffer, size_t buffer_size) {
     std::string sanitized_msg = error_message;
     for (auto& c : sanitized_msg) {
@@ -175,49 +224,58 @@ inline std::vector<ToolFunction> parse_tools_json(const std::string& json) {
     return tools;
 }
 
-inline void parse_options_json(const std::string& json, 
-                               float& temperature, float& top_p, 
+inline void parse_options_json(const std::string& json,
+                               float& temperature, float& top_p,
                                size_t& top_k, size_t& max_tokens,
-                               std::vector<std::string>& stop_sequences) {
+                               std::vector<std::string>& stop_sequences,
+                               bool& force_tools) {
     temperature = 0.0f;
-    top_p = 0.0f;       
-    top_k = 0;           
-    max_tokens = 100;    
+    top_p = 0.0f;
+    top_k = 0;
+    max_tokens = 100;
+    force_tools = false;
     stop_sequences.clear();
-    
+
     if (json.empty()) return;
-    
+
     size_t pos = json.find("\"temperature\"");
     if (pos != std::string::npos) {
         pos = json.find(':', pos) + 1;
         temperature = std::stof(json.substr(pos));
     }
-    
+
     pos = json.find("\"top_p\"");
     if (pos != std::string::npos) {
         pos = json.find(':', pos) + 1;
         top_p = std::stof(json.substr(pos));
     }
-    
+
     pos = json.find("\"top_k\"");
     if (pos != std::string::npos) {
         pos = json.find(':', pos) + 1;
         top_k = std::stoul(json.substr(pos));
     }
-    
+
     pos = json.find("\"max_tokens\"");
     if (pos != std::string::npos) {
         pos = json.find(':', pos) + 1;
         max_tokens = std::stoul(json.substr(pos));
     }
-    
+
+    pos = json.find("\"force_tools\"");
+    if (pos != std::string::npos) {
+        pos = json.find(':', pos) + 1;
+        while (pos < json.length() && std::isspace(json[pos])) pos++;
+        force_tools = (json.substr(pos, 4) == "true");
+    }
+
     pos = json.find("\"stop_sequences\"");
     if (pos != std::string::npos) {
         pos = json.find('[', pos);
         if (pos != std::string::npos) {
             size_t end_pos = json.find(']', pos);
             size_t seq_pos = json.find('"', pos);
-            
+
             while (seq_pos != std::string::npos && seq_pos < end_pos) {
                 size_t seq_start = seq_pos + 1;
                 size_t seq_end = json.find('"', seq_start);
@@ -234,7 +292,7 @@ inline std::string format_tools_for_prompt(const std::vector<ToolFunction>& tool
     if (tools.empty()) return "";
     std::string formatted_tools_json;
     for (size_t i = 0; i < tools.size(); i++) {
-        if (i > 0) formatted_tools_json += ",\n";
+        if (i > 0) formatted_tools_json += "\n";
         formatted_tools_json += "{\"type\":\"function\",\"function\":{\"name\":\""
                               + tools[i].name
                               + "\",\"description\":\""
@@ -253,11 +311,43 @@ inline void parse_function_calls_from_response(const std::string& response_text,
     regular_response = response_text;
     function_calls.clear();
 
+    gemma::parse_function_calls(regular_response, function_calls);
+
+    // Parse Qwen-style function calls: <tool_call>{"name": "...", "arguments": {...}}</tool_call>
+    const std::string QWEN_TOOL_START = "<tool_call>";
+    const std::string QWEN_TOOL_END = "</tool_call>";
+    size_t qwen_start_pos = 0;
+
+    while ((qwen_start_pos = regular_response.find(QWEN_TOOL_START, qwen_start_pos)) != std::string::npos) {
+        size_t content_start = qwen_start_pos + QWEN_TOOL_START.length();
+        size_t qwen_end_pos = regular_response.find(QWEN_TOOL_END, content_start);
+
+        if (qwen_end_pos != std::string::npos) {
+            std::string json_content = regular_response.substr(content_start, qwen_end_pos - content_start);
+
+            size_t first = json_content.find_first_not_of(" \t\n\r");
+            size_t last = json_content.find_last_not_of(" \t\n\r");
+            if (first != std::string::npos && last != std::string::npos) {
+                json_content = json_content.substr(first, last - first + 1);
+            }
+
+            if (json_content.size() > 2 && json_content[0] == '{' &&
+                json_content.find("\"name\"") != std::string::npos) {
+                function_calls.push_back(json_content);
+            }
+
+            regular_response.erase(qwen_start_pos, qwen_end_pos + QWEN_TOOL_END.length() - qwen_start_pos);
+        } else {
+            break;
+        }
+    }
+
+    // Parse LFM2-style function calls: <|tool_call_start|>[name(args)]<|tool_call_end|>
     const std::string TOOL_CALL_START = "<|tool_call_start|>";
     const std::string TOOL_CALL_END = "<|tool_call_end|>";
     size_t tool_start_pos = 0;
 
-    while ((tool_start_pos = response_text.find(TOOL_CALL_START, tool_start_pos)) != std::string::npos) {
+    while ((tool_start_pos = regular_response.find(TOOL_CALL_START, tool_start_pos)) != std::string::npos) {
         size_t content_start = tool_start_pos + TOOL_CALL_START.length();
         size_t tool_end_pos = response_text.find(TOOL_CALL_END, content_start);
 
@@ -398,4 +488,41 @@ inline std::string construct_response_json(const std::string& regular_response,
 } // namespace ffi
 } // namespace cactus
 
-#endif // CACTUS_FFI_UTILS_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+const char* cactus_get_last_error();
+
+__attribute__((weak))
+const char* register_app(const char* encrypted_data);
+
+__attribute__((weak))
+const char* get_device_id(const char* current_token);
+
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+
+__attribute__((weak))
+inline const char* register_app(const char* encrypted_data) {
+    (void)encrypted_data;
+    static thread_local std::string uuid_storage;
+    uuid_storage = cactus::ffi::generateUUID();
+    return uuid_storage.c_str();
+}
+
+__attribute__((weak))
+inline const char* get_device_id(const char* current_token) {
+    (void)current_token;
+    static thread_local std::string uuid_storage;
+    uuid_storage = cactus::ffi::generateUUID();
+    return uuid_storage.c_str();
+}
+}
+#endif
+
+#endif // CACTUS_UTILS_H

package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/engine.h

@@ -3,6 +3,7 @@
 #include <vector>
 #include <string>
 #include <unordered_map>
+#include <unordered_set>
 #include <memory>
 #include <cstdint>
 
@@ -32,6 +33,9 @@ extern "C" {
 class CactusGraph;
 
 namespace cactus {
+namespace npu {
+    class NPUPrefill;
+}
 namespace engine {
 
 class Siglip2Preprocessor;
@@ -127,9 +131,12 @@ struct MergeRule {
 struct ChatMessage {
     std::string role;
     std::string content;
+    std::string name;
     std::vector<std::string> images;
 };
 
+
+
 class Tokenizer {
 public:
     virtual ~Tokenizer() = default;
@@ -325,6 +332,8 @@ struct KVCache {
     struct LayerCache {
         std::vector<uint8_t> keys;
         std::vector<uint8_t> values;
+        std::vector<float> key_scales;   
+        std::vector<float> value_scales; 
     };
 
     std::vector<LayerCache> layer_caches;
@@ -349,7 +358,12 @@ struct KVCache {
     void update_from_graph(CactusGraph* gb, const std::vector<size_t>& k_nodes,
                           const std::vector<size_t>& v_nodes, size_t seq_len,
                           size_t num_layers, size_t kv_heads, size_t head_dim);
+
+    void update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data,
+                         size_t num_tokens, size_t kv_heads, size_t head_dim);
+
     bool is_empty() const { return current_seq_len == 0; }
+    bool is_int8() const { return precision == Precision::INT8; }
     void* get_key_ptr(size_t layer);
     void* get_value_ptr(size_t layer);
 
@@ -363,6 +377,100 @@ struct KVCache {
 
     CircularView get_key_view(size_t layer);
     CircularView get_value_view(size_t layer);
+
+    const int8_t* get_keys_int8(size_t layer) const;
+    const int8_t* get_values_int8(size_t layer) const;
+    const float* get_key_scales(size_t layer) const;
+    const float* get_value_scales(size_t layer) const;
+};
+
+class ToolCallConstrainer {
+public:
+    enum class State {
+        DONE,                   
+
+        QWEN_START,             
+        QWEN_EXPECT_OPEN_BRACE, 
+        QWEN_EXPECT_NAME_KEY, 
+        QWEN_EXPECT_NAME_COLON,
+        QWEN_EXPECT_NAME_VALUE,
+        QWEN_EXPECT_COMMA, 
+        QWEN_EXPECT_ARGS_KEY, 
+        QWEN_EXPECT_ARGS_COLON, 
+        QWEN_IN_ARGUMENTS,  
+        QWEN_EXPECT_CLOSE_BRACE,
+        QWEN_EXPECT_END, 
+
+        LFM_START,              
+        LFM_EXPECT_BRACKET, 
+        LFM_IN_FUNC_NAME,
+        LFM_EXPECT_PAREN,
+        LFM_IN_ARGUMENTS, 
+        LFM_EXPECT_BRACKET_CLOSE, 
+        LFM_EXPECT_END,   
+
+        GEMMA_START,           
+        GEMMA_EXPECT_CALL, 
+        GEMMA_IN_FUNC_NAME, 
+        GEMMA_EXPECT_BRACE, 
+        GEMMA_IN_ARGUMENTS, 
+        GEMMA_EXPECT_END 
+    };
+
+    void init(Config::ModelType model_type,
+              const std::vector<std::string>& function_names,
+              Tokenizer* tokenizer);
+
+    const std::unordered_map<uint32_t, float>& get_bias() const { return current_bias_; }
+
+    void update(uint32_t token_id, const std::string& decoded_text);
+
+    void reset();
+
+    bool is_active() const { return active_; }
+
+private:
+    bool active_ = false;
+    State state_ = State::QWEN_START;
+    Config::ModelType model_type_ = Config::ModelType::QWEN;
+    Tokenizer* tokenizer_ = nullptr;
+
+    std::vector<std::string> function_names_;
+    std::string generated_text_;
+    int brace_depth_ = 0;  
+
+    std::unordered_set<uint32_t> qwen_tool_call_start_tokens_; 
+    std::unordered_set<uint32_t> qwen_tool_call_end_tokens_;   
+    std::unordered_set<uint32_t> open_brace_tokens_;         
+    std::unordered_set<uint32_t> close_brace_tokens_;       
+    std::unordered_set<uint32_t> colon_tokens_;            
+    std::unordered_set<uint32_t> comma_tokens_;          
+    std::unordered_set<uint32_t> name_key_tokens_;           
+    std::unordered_set<uint32_t> args_key_tokens_;         
+    std::unordered_set<uint32_t> quote_tokens_;            
+    std::unordered_set<uint32_t> backtick_tokens_;   
+    std::unordered_set<uint32_t> all_func_name_tokens_;
+    std::unordered_map<std::string, std::vector<uint32_t>> func_name_sequences_;  
+
+    std::unordered_set<uint32_t> tool_start_tokens_;
+    std::unordered_set<uint32_t> tool_end_tokens_;
+    std::unordered_set<uint32_t> bracket_open_tokens_;   
+    std::unordered_set<uint32_t> bracket_close_tokens_;  
+    std::unordered_set<uint32_t> paren_open_tokens_;     
+    std::unordered_set<uint32_t> paren_close_tokens_;   
+    std::unordered_set<uint32_t> equals_tokens_;        
+
+    std::unordered_set<uint32_t> gemma_call_start_tokens_;    
+    std::unordered_set<uint32_t> gemma_call_end_tokens_;       
+    std::unordered_set<uint32_t> gemma_response_start_tokens_; 
+    std::unordered_set<uint32_t> gemma_call_prefix_tokens_;    
+    std::unordered_set<uint32_t> escape_tokens_;              
+
+    std::unordered_map<uint32_t, float> current_bias_;
+
+    void compute_bias();
+    void tokenize_grammar_elements();
+    void add_tokens_for_string(const std::string& str, std::unordered_set<uint32_t>& token_set);
 };
 
 class Model {
@@ -386,26 +494,40 @@ public:
     virtual bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size,
               const std::string& system_prompt = "", bool do_warmup = true);
 
-    virtual uint32_t generate(const std::vector<uint32_t>& tokens, float temperature = -1.0f, float top_p = -1.0f,
-                      size_t top_k = 0, const std::string& profile_file = "", bool prefill_only = false);
+    virtual uint32_t decode(const std::vector<uint32_t>& tokens, float temperature = -1.0f, float top_p = -1.0f,
+                      size_t top_k = 0, const std::string& profile_file = "");
+
+    virtual void prefill(const std::vector<uint32_t>& tokens, size_t chunk_size = 256, const std::string& profile_file = "");
 
-    virtual uint32_t generate_with_images(const std::vector<uint32_t>& tokens, const std::vector<std::string>& image_paths,
+    virtual uint32_t decode_with_images(const std::vector<uint32_t>& tokens, const std::vector<std::string>& image_paths,
                                           float temperature = -1.0f, float top_p = -1.0f,
                                           size_t top_k = 0, const std::string& profile_file = "");
-    
-    virtual uint32_t generate_with_audio(const std::vector<uint32_t>& tokens, const std::vector<float>& mel_bins, float temperature = 0.0f, float top_p = 0.0f,
+
+    virtual uint32_t decode_with_audio(const std::vector<uint32_t>& tokens, const std::vector<float>& mel_bins, float temperature = 0.0f, float top_p = 0.0f,
                       size_t top_k = 0, const std::string& profile_file = "");
 
-    std::vector<float> get_embeddings(const std::vector<uint32_t>& tokens, bool pooled = true, const std::string& profile_file = "");
+    std::vector<float> get_embeddings(const std::vector<uint32_t>& tokens, bool pooled = true, bool normalize = false, const std::string& profile_file = "");
     
     virtual std::vector<float> get_image_embeddings(const std::string& image_path);
     
     virtual std::vector<float> get_audio_embeddings(const std::vector<float>& mel_bins);
 
     virtual void reset_cache() { kv_cache_.reset(); }
-    
+
+    double score_tokens_window_logprob(const std::vector<uint32_t>& tokens, size_t start, size_t end, size_t context, size_t* tokens_scored);
+
+
+
     void set_cache_window(size_t window_size, size_t sink_size = 4) { kv_cache_.set_window_size(window_size, sink_size); }
 
+    bool load_npu_prefill(const std::string& model_path);
+    bool has_npu_prefill() const;
+    size_t get_prefill_chunk_size() const;
+
+    void set_tool_constraints(const std::vector<std::string>& function_names);
+    void clear_tool_constraints();
+    void update_tool_constraints(uint32_t token_id);
+
     void* graph_handle_;
 
 protected:
@@ -449,6 +571,12 @@ protected:
     bool init_internal(CactusGraph* gb, const std::string& model_folder, size_t context_size,
                        const std::string& system_prompt, bool do_warmup);
     bool owns_graph_;
+
+    std::unique_ptr<npu::NPUPrefill> npu_prefill_;
+    void prefill_npu(const std::vector<uint32_t>& tokens);
+    virtual std::vector<__fp16> get_token_embeddings(const std::vector<uint32_t>& tokens);
+
+    ToolCallConstrainer tool_constrainer_;
 };
 
 std::unique_ptr<Model> create_model(const std::string& model_folder);