cactus-react-native 1.4.0 → 1.5.0

package/ios/cactus.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_ffi.h

@@ -67,6 +67,30 @@ CACTUS_FFI_EXPORT int cactus_transcribe(
     size_t pcm_buffer_size
 );
 
+typedef void* cactus_stream_transcribe_t;
+
+CACTUS_FFI_EXPORT cactus_stream_transcribe_t cactus_stream_transcribe_init(cactus_model_t model);
+
+CACTUS_FFI_EXPORT int cactus_stream_transcribe_insert(
+    cactus_stream_transcribe_t stream,
+    const uint8_t* pcm_buffer,
+    size_t pcm_buffer_size
+);
+
+CACTUS_FFI_EXPORT int cactus_stream_transcribe_process(
+    cactus_stream_transcribe_t stream,
+    char* response_buffer,
+    size_t buffer_size,
+    const char* options_json
+);
+
+CACTUS_FFI_EXPORT int cactus_stream_transcribe_finalize(
+    cactus_stream_transcribe_t stream,
+    char* response_buffer,
+    size_t buffer_size
+);
+
+CACTUS_FFI_EXPORT void cactus_stream_transcribe_destroy(cactus_stream_transcribe_t stream);
 
 CACTUS_FFI_EXPORT int cactus_embed(
     cactus_model_t model,

package/ios/cactus.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_utils.h

@@ -63,6 +63,14 @@ struct ToolFunction {
     std::unordered_map<std::string, std::string> parameters;
 };
 
+} // namespace ffi
+} // namespace cactus
+
+#include "gemma_tools.h"
+
+namespace cactus {
+namespace ffi {
+
 inline void handle_error_response(const std::string& error_message, char* response_buffer, size_t buffer_size) {
     std::string sanitized_msg = error_message;
     for (auto& c : sanitized_msg) {
@@ -303,11 +311,43 @@ inline void parse_function_calls_from_response(const std::string& response_text,
     regular_response = response_text;
     function_calls.clear();
 
+    gemma::parse_function_calls(regular_response, function_calls);
+
+    // Parse Qwen-style function calls: <tool_call>{"name": "...", "arguments": {...}}</tool_call>
+    const std::string QWEN_TOOL_START = "<tool_call>";
+    const std::string QWEN_TOOL_END = "</tool_call>";
+    size_t qwen_start_pos = 0;
+
+    while ((qwen_start_pos = regular_response.find(QWEN_TOOL_START, qwen_start_pos)) != std::string::npos) {
+        size_t content_start = qwen_start_pos + QWEN_TOOL_START.length();
+        size_t qwen_end_pos = regular_response.find(QWEN_TOOL_END, content_start);
+
+        if (qwen_end_pos != std::string::npos) {
+            std::string json_content = regular_response.substr(content_start, qwen_end_pos - content_start);
+
+            size_t first = json_content.find_first_not_of(" \t\n\r");
+            size_t last = json_content.find_last_not_of(" \t\n\r");
+            if (first != std::string::npos && last != std::string::npos) {
+                json_content = json_content.substr(first, last - first + 1);
+            }
+
+            if (json_content.size() > 2 && json_content[0] == '{' &&
+                json_content.find("\"name\"") != std::string::npos) {
+                function_calls.push_back(json_content);
+            }
+
+            regular_response.erase(qwen_start_pos, qwen_end_pos + QWEN_TOOL_END.length() - qwen_start_pos);
+        } else {
+            break;
+        }
+    }
+
+    // Parse LFM2-style function calls: <|tool_call_start|>[name(args)]<|tool_call_end|>
     const std::string TOOL_CALL_START = "<|tool_call_start|>";
     const std::string TOOL_CALL_END = "<|tool_call_end|>";
     size_t tool_start_pos = 0;
 
-    while ((tool_start_pos = response_text.find(TOOL_CALL_START, tool_start_pos)) != std::string::npos) {
+    while ((tool_start_pos = regular_response.find(TOOL_CALL_START, tool_start_pos)) != std::string::npos) {
         size_t content_start = tool_start_pos + TOOL_CALL_START.length();
         size_t tool_end_pos = response_text.find(TOOL_CALL_END, content_start);
 

package/ios/cactus.xcframework/ios-arm64-simulator/cactus.framework/Headers/engine.h

@@ -131,9 +131,12 @@ struct MergeRule {
 struct ChatMessage {
     std::string role;
     std::string content;
+    std::string name;
     std::vector<std::string> images;
 };
 
+
+
 class Tokenizer {
 public:
     virtual ~Tokenizer() = default;
@@ -329,6 +332,8 @@ struct KVCache {
     struct LayerCache {
         std::vector<uint8_t> keys;
         std::vector<uint8_t> values;
+        std::vector<float> key_scales;   
+        std::vector<float> value_scales; 
     };
 
     std::vector<LayerCache> layer_caches;
@@ -354,13 +359,11 @@ struct KVCache {
                           const std::vector<size_t>& v_nodes, size_t seq_len,
                           size_t num_layers, size_t kv_heads, size_t head_dim);
 
-    // Update KV cache from NPU prefill outputs
-    // NPU outputs are in shape [num_tokens, num_kv_heads, head_dim]
-    // This handles transposition to cache format and sliding window
     void update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data,
                          size_t num_tokens, size_t kv_heads, size_t head_dim);
 
     bool is_empty() const { return current_seq_len == 0; }
+    bool is_int8() const { return precision == Precision::INT8; }
     void* get_key_ptr(size_t layer);
     void* get_value_ptr(size_t layer);
 
@@ -374,33 +377,44 @@ struct KVCache {
 
     CircularView get_key_view(size_t layer);
     CircularView get_value_view(size_t layer);
+
+    const int8_t* get_keys_int8(size_t layer) const;
+    const int8_t* get_values_int8(size_t layer) const;
+    const float* get_key_scales(size_t layer) const;
+    const float* get_value_scales(size_t layer) const;
 };
 
 class ToolCallConstrainer {
 public:
     enum class State {
-        START,                  // -> expect {
-        EXPECT_FC_KEY,          // -> expect "function_call"
-        EXPECT_FC_COLON,        // -> expect :
-        EXPECT_FC_OPEN_BRACE,   // -> expect {
-        EXPECT_NAME_KEY,        // -> expect "name"
-        EXPECT_NAME_COLON,      // -> expect :
-        EXPECT_NAME_VALUE,      // -> expect "<function_name>"
-        EXPECT_COMMA,           // -> expect ,
-        EXPECT_ARGS_KEY,        // -> expect "arguments"
-        EXPECT_ARGS_COLON,      // -> expect :
-        IN_ARGUMENTS,           // -> free JSON, track brace depth
-        EXPECT_INNER_CLOSE,     // -> expect } to close inner object
-        EXPECT_OUTER_CLOSE,     // -> expect } to close outer object
-        DONE,                   // complete
-
-        LFM_START,              // -> expect <|tool_call_start|>
-        LFM_EXPECT_BRACKET,     // -> expect [
-        LFM_IN_FUNC_NAME,       // -> expect function name
-        LFM_EXPECT_PAREN,       // -> expect (
-        LFM_IN_ARGUMENTS,       // -> arguments until )
-        LFM_EXPECT_BRACKET_CLOSE, // -> expect ]
-        LFM_EXPECT_END          // -> expect <|tool_call_end|>
+        DONE,                   
+
+        QWEN_START,             
+        QWEN_EXPECT_OPEN_BRACE, 
+        QWEN_EXPECT_NAME_KEY, 
+        QWEN_EXPECT_NAME_COLON,
+        QWEN_EXPECT_NAME_VALUE,
+        QWEN_EXPECT_COMMA, 
+        QWEN_EXPECT_ARGS_KEY, 
+        QWEN_EXPECT_ARGS_COLON, 
+        QWEN_IN_ARGUMENTS,  
+        QWEN_EXPECT_CLOSE_BRACE,
+        QWEN_EXPECT_END, 
+
+        LFM_START,              
+        LFM_EXPECT_BRACKET, 
+        LFM_IN_FUNC_NAME,
+        LFM_EXPECT_PAREN,
+        LFM_IN_ARGUMENTS, 
+        LFM_EXPECT_BRACKET_CLOSE, 
+        LFM_EXPECT_END,   
+
+        GEMMA_START,           
+        GEMMA_EXPECT_CALL, 
+        GEMMA_IN_FUNC_NAME, 
+        GEMMA_EXPECT_BRACE, 
+        GEMMA_IN_ARGUMENTS, 
+        GEMMA_EXPECT_END 
     };
 
     void init(Config::ModelType model_type,
@@ -417,36 +431,40 @@ public:
 
 private:
     bool active_ = false;
-    State state_ = State::START;
+    State state_ = State::QWEN_START;
     Config::ModelType model_type_ = Config::ModelType::QWEN;
     Tokenizer* tokenizer_ = nullptr;
 
     std::vector<std::string> function_names_;
     std::string generated_text_;
-    int brace_depth_ = 0;  // Track nested braces in arguments
-
-    // Pre-tokenized token sets for each grammar element
-    std::unordered_set<uint32_t> open_brace_tokens_;      // {
-    std::unordered_set<uint32_t> close_brace_tokens_;     // }
-    std::unordered_set<uint32_t> colon_tokens_;           // :
-    std::unordered_set<uint32_t> comma_tokens_;           // ,
-    std::unordered_set<uint32_t> fc_key_tokens_;          // "function_call"
-    std::unordered_set<uint32_t> name_key_tokens_;        // "name"
-    std::unordered_set<uint32_t> args_key_tokens_;        // "arguments"
-    std::unordered_set<uint32_t> quote_tokens_;           // "
-    std::unordered_set<uint32_t> backtick_tokens_;        // ` (to block markdown code fences)
-    std::unordered_set<uint32_t> response_starter_tokens_; // Common response starters to block (I, I'm, Sorry, etc.)
-    std::unordered_set<uint32_t> all_func_name_tokens_;   // All function name tokens combined
-    std::unordered_map<std::string, std::vector<uint32_t>> func_name_sequences_;  // Full token sequence per function
-
-    // LFM2-specific tokens
+    int brace_depth_ = 0;  
+
+    std::unordered_set<uint32_t> qwen_tool_call_start_tokens_; 
+    std::unordered_set<uint32_t> qwen_tool_call_end_tokens_;   
+    std::unordered_set<uint32_t> open_brace_tokens_;         
+    std::unordered_set<uint32_t> close_brace_tokens_;       
+    std::unordered_set<uint32_t> colon_tokens_;            
+    std::unordered_set<uint32_t> comma_tokens_;          
+    std::unordered_set<uint32_t> name_key_tokens_;           
+    std::unordered_set<uint32_t> args_key_tokens_;         
+    std::unordered_set<uint32_t> quote_tokens_;            
+    std::unordered_set<uint32_t> backtick_tokens_;   
+    std::unordered_set<uint32_t> all_func_name_tokens_;
+    std::unordered_map<std::string, std::vector<uint32_t>> func_name_sequences_;  
+
     std::unordered_set<uint32_t> tool_start_tokens_;
     std::unordered_set<uint32_t> tool_end_tokens_;
-    std::unordered_set<uint32_t> bracket_open_tokens_;    // [
-    std::unordered_set<uint32_t> bracket_close_tokens_;   // ]
-    std::unordered_set<uint32_t> paren_open_tokens_;      // (
-    std::unordered_set<uint32_t> paren_close_tokens_;     // )
-    std::unordered_set<uint32_t> equals_tokens_;          // =
+    std::unordered_set<uint32_t> bracket_open_tokens_;   
+    std::unordered_set<uint32_t> bracket_close_tokens_;  
+    std::unordered_set<uint32_t> paren_open_tokens_;     
+    std::unordered_set<uint32_t> paren_close_tokens_;   
+    std::unordered_set<uint32_t> equals_tokens_;        
+
+    std::unordered_set<uint32_t> gemma_call_start_tokens_;    
+    std::unordered_set<uint32_t> gemma_call_end_tokens_;       
+    std::unordered_set<uint32_t> gemma_response_start_tokens_; 
+    std::unordered_set<uint32_t> gemma_call_prefix_tokens_;    
+    std::unordered_set<uint32_t> escape_tokens_;              
 
     std::unordered_map<uint32_t, float> current_bias_;