cactus-react-native 1.4.0 → 1.5.0

package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/engine.h

@@ -131,9 +131,12 @@ struct MergeRule {
 struct ChatMessage {
     std::string role;
     std::string content;
+    std::string name;
     std::vector<std::string> images;
 };
 
+
+
 class Tokenizer {
 public:
     virtual ~Tokenizer() = default;
@@ -329,6 +332,8 @@ struct KVCache {
     struct LayerCache {
         std::vector<uint8_t> keys;
         std::vector<uint8_t> values;
+        std::vector<float> key_scales;   
+        std::vector<float> value_scales; 
     };
 
     std::vector<LayerCache> layer_caches;
@@ -354,13 +359,11 @@ struct KVCache {
                           const std::vector<size_t>& v_nodes, size_t seq_len,
                           size_t num_layers, size_t kv_heads, size_t head_dim);
 
-    // Update KV cache from NPU prefill outputs
-    // NPU outputs are in shape [num_tokens, num_kv_heads, head_dim]
-    // This handles transposition to cache format and sliding window
     void update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data,
                          size_t num_tokens, size_t kv_heads, size_t head_dim);
 
     bool is_empty() const { return current_seq_len == 0; }
+    bool is_int8() const { return precision == Precision::INT8; }
     void* get_key_ptr(size_t layer);
     void* get_value_ptr(size_t layer);
 
@@ -374,33 +377,44 @@ struct KVCache {
 
     CircularView get_key_view(size_t layer);
     CircularView get_value_view(size_t layer);
+
+    const int8_t* get_keys_int8(size_t layer) const;
+    const int8_t* get_values_int8(size_t layer) const;
+    const float* get_key_scales(size_t layer) const;
+    const float* get_value_scales(size_t layer) const;
 };
 
 class ToolCallConstrainer {
 public:
     enum class State {
-        START,                  // -> expect {
-        EXPECT_FC_KEY,          // -> expect "function_call"
-        EXPECT_FC_COLON,        // -> expect :
-        EXPECT_FC_OPEN_BRACE,   // -> expect {
-        EXPECT_NAME_KEY,        // -> expect "name"
-        EXPECT_NAME_COLON,      // -> expect :
-        EXPECT_NAME_VALUE,      // -> expect "<function_name>"
-        EXPECT_COMMA,           // -> expect ,
-        EXPECT_ARGS_KEY,        // -> expect "arguments"
-        EXPECT_ARGS_COLON,      // -> expect :
-        IN_ARGUMENTS,           // -> free JSON, track brace depth
-        EXPECT_INNER_CLOSE,     // -> expect } to close inner object
-        EXPECT_OUTER_CLOSE,     // -> expect } to close outer object
-        DONE,                   // complete
-
-        LFM_START,              // -> expect <|tool_call_start|>
-        LFM_EXPECT_BRACKET,     // -> expect [
-        LFM_IN_FUNC_NAME,       // -> expect function name
-        LFM_EXPECT_PAREN,       // -> expect (
-        LFM_IN_ARGUMENTS,       // -> arguments until )
-        LFM_EXPECT_BRACKET_CLOSE, // -> expect ]
-        LFM_EXPECT_END          // -> expect <|tool_call_end|>
+        DONE,                   
+
+        QWEN_START,             
+        QWEN_EXPECT_OPEN_BRACE, 
+        QWEN_EXPECT_NAME_KEY, 
+        QWEN_EXPECT_NAME_COLON,
+        QWEN_EXPECT_NAME_VALUE,
+        QWEN_EXPECT_COMMA, 
+        QWEN_EXPECT_ARGS_KEY, 
+        QWEN_EXPECT_ARGS_COLON, 
+        QWEN_IN_ARGUMENTS,  
+        QWEN_EXPECT_CLOSE_BRACE,
+        QWEN_EXPECT_END, 
+
+        LFM_START,              
+        LFM_EXPECT_BRACKET, 
+        LFM_IN_FUNC_NAME,
+        LFM_EXPECT_PAREN,
+        LFM_IN_ARGUMENTS, 
+        LFM_EXPECT_BRACKET_CLOSE, 
+        LFM_EXPECT_END,   
+
+        GEMMA_START,           
+        GEMMA_EXPECT_CALL, 
+        GEMMA_IN_FUNC_NAME, 
+        GEMMA_EXPECT_BRACE, 
+        GEMMA_IN_ARGUMENTS, 
+        GEMMA_EXPECT_END 
     };
 
     void init(Config::ModelType model_type,
@@ -417,36 +431,40 @@ public:
 
 private:
     bool active_ = false;
-    State state_ = State::START;
+    State state_ = State::QWEN_START;
     Config::ModelType model_type_ = Config::ModelType::QWEN;
     Tokenizer* tokenizer_ = nullptr;
 
     std::vector<std::string> function_names_;
     std::string generated_text_;
-    int brace_depth_ = 0;  // Track nested braces in arguments
-
-    // Pre-tokenized token sets for each grammar element
-    std::unordered_set<uint32_t> open_brace_tokens_;      // {
-    std::unordered_set<uint32_t> close_brace_tokens_;     // }
-    std::unordered_set<uint32_t> colon_tokens_;           // :
-    std::unordered_set<uint32_t> comma_tokens_;           // ,
-    std::unordered_set<uint32_t> fc_key_tokens_;          // "function_call"
-    std::unordered_set<uint32_t> name_key_tokens_;        // "name"
-    std::unordered_set<uint32_t> args_key_tokens_;        // "arguments"
-    std::unordered_set<uint32_t> quote_tokens_;           // "
-    std::unordered_set<uint32_t> backtick_tokens_;        // ` (to block markdown code fences)
-    std::unordered_set<uint32_t> response_starter_tokens_; // Common response starters to block (I, I'm, Sorry, etc.)
-    std::unordered_set<uint32_t> all_func_name_tokens_;   // All function name tokens combined
-    std::unordered_map<std::string, std::vector<uint32_t>> func_name_sequences_;  // Full token sequence per function
-
-    // LFM2-specific tokens
+    int brace_depth_ = 0;  
+
+    std::unordered_set<uint32_t> qwen_tool_call_start_tokens_; 
+    std::unordered_set<uint32_t> qwen_tool_call_end_tokens_;   
+    std::unordered_set<uint32_t> open_brace_tokens_;         
+    std::unordered_set<uint32_t> close_brace_tokens_;       
+    std::unordered_set<uint32_t> colon_tokens_;            
+    std::unordered_set<uint32_t> comma_tokens_;          
+    std::unordered_set<uint32_t> name_key_tokens_;           
+    std::unordered_set<uint32_t> args_key_tokens_;         
+    std::unordered_set<uint32_t> quote_tokens_;            
+    std::unordered_set<uint32_t> backtick_tokens_;   
+    std::unordered_set<uint32_t> all_func_name_tokens_;
+    std::unordered_map<std::string, std::vector<uint32_t>> func_name_sequences_;  
+
     std::unordered_set<uint32_t> tool_start_tokens_;
     std::unordered_set<uint32_t> tool_end_tokens_;
-    std::unordered_set<uint32_t> bracket_open_tokens_;    // [
-    std::unordered_set<uint32_t> bracket_close_tokens_;   // ]
-    std::unordered_set<uint32_t> paren_open_tokens_;      // (
-    std::unordered_set<uint32_t> paren_close_tokens_;     // )
-    std::unordered_set<uint32_t> equals_tokens_;          // =
+    std::unordered_set<uint32_t> bracket_open_tokens_;   
+    std::unordered_set<uint32_t> bracket_close_tokens_;  
+    std::unordered_set<uint32_t> paren_open_tokens_;     
+    std::unordered_set<uint32_t> paren_close_tokens_;   
+    std::unordered_set<uint32_t> equals_tokens_;        
+
+    std::unordered_set<uint32_t> gemma_call_start_tokens_;    
+    std::unordered_set<uint32_t> gemma_call_end_tokens_;       
+    std::unordered_set<uint32_t> gemma_response_start_tokens_; 
+    std::unordered_set<uint32_t> gemma_call_prefix_tokens_;    
+    std::unordered_set<uint32_t> escape_tokens_;              
 
     std::unordered_map<uint32_t, float> current_bias_;