cactus-react-native 1.0.1 → 1.1.0

package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/ffi_utils.h

@@ -8,6 +8,10 @@
 #include <stdexcept>
 #include <sstream>
 #include <iomanip>
+#include <fstream>
+#include <iostream>
+#include <filesystem>
+#include <cctype>
 
 namespace cactus {
 namespace ffi {
@@ -30,8 +34,10 @@ inline void handle_error_response(const std::string& error_message, char* respon
     }
 }
 
-inline std::vector<cactus::engine::ChatMessage> parse_messages_json(const std::string& json) {
+inline std::vector<cactus::engine::ChatMessage> parse_messages_json(const std::string& json, 
+                                                                   std::vector<std::string>& out_image_paths) {
     std::vector<cactus::engine::ChatMessage> messages;
+    out_image_paths.clear();
     
     size_t pos = json.find('[');
     if (pos == std::string::npos) {
@@ -42,42 +48,79 @@ inline std::vector<cactus::engine::ChatMessage> parse_messages_json(const std::s
     while (pos != std::string::npos) {
         cactus::engine::ChatMessage msg;
         
+        size_t obj_start = pos;
+        int brace_count = 1;
+        size_t obj_end = obj_start + 1;
+        while (obj_end < json.length() && brace_count > 0) {
+            if (json[obj_end] == '{') brace_count++;
+            else if (json[obj_end] == '}') brace_count--;
+            obj_end++;
+        }
+
         size_t role_pos = json.find("\"role\"", pos);
-        if (role_pos == std::string::npos) break;
+        if (role_pos == std::string::npos || role_pos >= obj_end) break;
         
         size_t role_start = json.find('"', role_pos + 6) + 1;
         size_t role_end = json.find('"', role_start);
         msg.role = json.substr(role_start, role_end - role_start);
         
         size_t content_pos = json.find("\"content\"", role_end);
-        if (content_pos == std::string::npos) break;
-        
-        size_t content_start = json.find('"', content_pos + 9) + 1;
-        size_t content_end = content_start;
-        
-        while (content_end < json.length()) {
-            content_end = json.find('"', content_end);
-            if (content_end == std::string::npos) break;
-            if (json[content_end - 1] != '\\') break;
-            content_end++;
+        if (content_pos != std::string::npos && content_pos < obj_end) {
+            size_t content_start = json.find('"', content_pos + 9) + 1;
+            size_t content_end = content_start;
+            
+            while (content_end < json.length()) {
+                content_end = json.find('"', content_end);
+                if (content_end == std::string::npos) break;
+                if (json[content_end - 1] != '\\') break;
+                content_end++;
+            }
+            
+            msg.content = json.substr(content_start, content_end - content_start);
+            
+            size_t escape_pos = 0;
+            while ((escape_pos = msg.content.find("\\n", escape_pos)) != std::string::npos) {
+                msg.content.replace(escape_pos, 2, "\n");
+                escape_pos += 1;
+            }
+            escape_pos = 0;
+            while ((escape_pos = msg.content.find("\\\"", escape_pos)) != std::string::npos) {
+                msg.content.replace(escape_pos, 2, "\"");
+                escape_pos += 1;
+            }
         }
         
-        msg.content = json.substr(content_start, content_end - content_start);
-        
-        size_t escape_pos = 0;
-        while ((escape_pos = msg.content.find("\\n", escape_pos)) != std::string::npos) {
-            msg.content.replace(escape_pos, 2, "\n");
-            escape_pos += 1;
-        }
-        escape_pos = 0;
-        while ((escape_pos = msg.content.find("\\\"", escape_pos)) != std::string::npos) {
-            msg.content.replace(escape_pos, 2, "\"");
-            escape_pos += 1;
+        size_t images_pos = json.find("\"images\"", pos);
+        if (images_pos != std::string::npos && images_pos < obj_end) {
+            size_t array_start = json.find('[', images_pos);
+            if (array_start != std::string::npos && array_start < obj_end) {
+                size_t array_end = json.find(']', array_start);
+                if (array_end != std::string::npos && array_end < obj_end) {
+                    size_t img_pos = array_start;
+                    while (true) {
+                        img_pos = json.find('"', img_pos + 1);
+                        if (img_pos == std::string::npos || img_pos >= array_end) break;
+                        
+                        size_t img_start = img_pos + 1;
+                        size_t img_end = json.find('"', img_start);
+                        if (img_end == std::string::npos || img_end > array_end) break;
+                        
+                        std::string img_path = json.substr(img_start, img_end - img_start);
+                        
+                        std::filesystem::path p(img_path);
+                        img_path = std::filesystem::absolute(p).string();
+                        
+                        msg.images.push_back(img_path);
+                        out_image_paths.push_back(img_path);
+                        img_pos = img_end;
+                    }
+                }
+            }
         }
         
         messages.push_back(msg);
         
-        pos = json.find('{', content_end);
+        pos = json.find('{', obj_end);
     }
     
     return messages;
@@ -136,10 +179,10 @@ inline void parse_options_json(const std::string& json,
                                float& temperature, float& top_p, 
                                size_t& top_k, size_t& max_tokens,
                                std::vector<std::string>& stop_sequences) {
-    temperature = -1.0f; // Use model default
-    top_p = -1.0f;       // Use model default
-    top_k = 0;           // Use model default
-    max_tokens = 100;    // FFI-level default
+    temperature = 0.0f;
+    top_p = 0.0f;       
+    top_k = 0;           
+    max_tokens = 100;    
     stop_sequences.clear();
     
     if (json.empty()) return;
@@ -192,47 +235,119 @@ inline std::string format_tools_for_prompt(const std::vector<ToolFunction>& tool
     std::string formatted_tools_json;
     for (size_t i = 0; i < tools.size(); i++) {
         if (i > 0) formatted_tools_json += ",\n";
-        formatted_tools_json += "  {\n";
-        formatted_tools_json += "    \"type\": \"function\",\n";
-        formatted_tools_json += "    \"function\": {\n";
-        formatted_tools_json += "      \"name\": \"" + tools[i].name + "\",\n";
-        formatted_tools_json += "      \"description\": \"" + tools[i].description + "\"";
+        formatted_tools_json += "{\"type\":\"function\",\"function\":{\"name\":\""
+                              + tools[i].name
+                              + "\",\"description\":\""
+                              + tools[i].description + "\"";
         if (tools[i].parameters.find("schema") != tools[i].parameters.end()) {
-            formatted_tools_json += ",\n      \"parameters\": " + tools[i].parameters.at("schema");
+            formatted_tools_json += ",\"parameters\":" + tools[i].parameters.at("schema");
         }
-        formatted_tools_json += "\n    }\n  }";
+        formatted_tools_json += "}}";
     }
     return formatted_tools_json;
 }
 
-inline void parse_function_calls_from_response(const std::string& response_text, 
-                                               std::string& regular_response, 
+inline void parse_function_calls_from_response(const std::string& response_text,
+                                               std::string& regular_response,
                                                std::vector<std::string>& function_calls) {
     regular_response = response_text;
     function_calls.clear();
 
+    const std::string TOOL_CALL_START = "<|tool_call_start|>";
+    const std::string TOOL_CALL_END = "<|tool_call_end|>";
+    size_t tool_start_pos = 0;
+
+    while ((tool_start_pos = response_text.find(TOOL_CALL_START, tool_start_pos)) != std::string::npos) {
+        size_t content_start = tool_start_pos + TOOL_CALL_START.length();
+        size_t tool_end_pos = response_text.find(TOOL_CALL_END, content_start);
+
+        if (tool_end_pos != std::string::npos) {
+            std::string tool_content = response_text.substr(content_start, tool_end_pos - content_start);
+
+            if (tool_content.size() > 2 && tool_content[0] == '[' && tool_content[tool_content.size()-1] == ']') {
+                tool_content = tool_content.substr(1, tool_content.size() - 2); 
+
+                size_t paren_pos = tool_content.find('(');
+                if (paren_pos != std::string::npos) {
+                    std::string func_name = tool_content.substr(0, paren_pos);
+                    std::string args_str = tool_content.substr(paren_pos + 1);
+
+                    if (!args_str.empty() && args_str.back() == ')') {
+                        args_str.pop_back();
+                    }
+
+                    std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":{";
+
+                    size_t arg_pos = 0;
+                    bool first_arg = true;
+                    while (arg_pos < args_str.length()) {
+                        while (arg_pos < args_str.length() && std::isspace(args_str[arg_pos])) arg_pos++;
+
+                        size_t eq_pos = args_str.find('=', arg_pos);
+                        if (eq_pos == std::string::npos) break;
+
+                        std::string arg_name = args_str.substr(arg_pos, eq_pos - arg_pos);
+
+                        size_t val_start = eq_pos + 1;
+                        size_t val_end = val_start;
+
+                        if (val_start < args_str.length() && args_str[val_start] == '"') {
+                            val_start++;
+                            val_end = args_str.find('"', val_start);
+                            if (val_end == std::string::npos) break;
+                        } else {
+                            val_end = args_str.find(',', val_start);
+                            if (val_end == std::string::npos) val_end = args_str.length();
+                        }
+
+                        std::string arg_value = args_str.substr(val_start, val_end - val_start);
+
+                        if (!first_arg) json_call += ",";
+                        json_call += "\"" + arg_name + "\":\"" + arg_value + "\"";
+                        first_arg = false;
+
+                        arg_pos = args_str.find(',', val_end);
+                        if (arg_pos != std::string::npos) {
+                            arg_pos++;
+                        } else {
+                            break;
+                        }
+                    }
+
+                    json_call += "}}";
+                    function_calls.push_back(json_call);
+                }
+            }
+
+            regular_response.erase(tool_start_pos, tool_end_pos + TOOL_CALL_END.length() - tool_start_pos);
+            tool_start_pos = tool_end_pos + TOOL_CALL_END.length();
+        } else {
+            break;
+        }
+    }
+
     const char* FUNCTION_CALL_MARKER = "\"function_call\"";
     size_t search_pos = 0;
-    const size_t text_len = response_text.length();
+    const size_t text_len = regular_response.length();
 
     while (search_pos < text_len) {
-        size_t marker_pos = response_text.find(FUNCTION_CALL_MARKER, search_pos);
+        size_t marker_pos = regular_response.find(FUNCTION_CALL_MARKER, search_pos);
         if (marker_pos == std::string::npos) break;
 
-        size_t json_start = response_text.find('{', marker_pos);
+        size_t json_start = regular_response.find('{', marker_pos);
         if (json_start == std::string::npos) break;
 
         int brace_count = 1;
         size_t json_end = json_start + 1;
         while (json_end < text_len && brace_count > 0) {
-            char c = response_text[json_end];
+            char c = regular_response[json_end];
             brace_count += (c == '{') - (c == '}');
             json_end++;
         }
 
         if (brace_count == 0) {
-            function_calls.push_back(response_text.substr(json_start, json_end - json_start));
-            regular_response = response_text.substr(0, marker_pos); 
+            function_calls.push_back(regular_response.substr(json_start, json_end - json_start));
+            regular_response = regular_response.substr(0, marker_pos);
             size_t last_bracket = regular_response.rfind('{');
             if(last_bracket != std::string::npos) {
                 regular_response = regular_response.substr(0, last_bracket);

package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/graph.h

@@ -28,10 +28,11 @@ enum class OpType {
     INPUT, PRECISION_CAST,
     ADD, ADD_CLIPPED, SUBTRACT, MULTIPLY, DIVIDE,
     MATMUL, TRANSPOSE, RESHAPE, SLICE, GATHER, EMBEDDING,
+    BILINEAR_INTERPOLATION,
     SUM, MEAN, VARIANCE, MIN, MAX,
-    RMS_NORM, ROPE, SOFTMAX, ATTENTION, CONV1D_CAUSAL,
+    RMS_NORM, ROPE, SOFTMAX, ATTENTION, CONV1D_CAUSAL, CONV1D_K3,
     SCALAR_ADD, SCALAR_SUBTRACT, SCALAR_MULTIPLY, SCALAR_DIVIDE, SCALAR_EXP, SCALAR_SQRT, SCALAR_COS, SCALAR_SIN,
-    SILU, GELU,
+    SILU, GELU, GELU_ERF,
     SAMPLE, CONCAT,
     SCATTER_TOPK,
     TOPK, LAYERNORM,
@@ -139,6 +140,7 @@ struct OpParams {
     ComputeBackend backend = ComputeBackend::CPU;
 
     size_t dilation = 1;
+    size_t stride = 1;
     float temperature = 1.0f;
     float top_p = 1.0f;
     size_t top_k = 0;
@@ -146,6 +148,8 @@ struct OpParams {
     
     size_t index_value = 0;  // For INDEX operation
     size_t num_classes = 0;  // For scatter operations
+    size_t dst_height = 0;
+    size_t dst_width = 0;   
 };
 
 struct GraphNode {
@@ -187,6 +191,12 @@ namespace ValidationUtils {
 class CactusGraph {
 public:
     CactusGraph();
+
+    struct DebugNodeEntry {
+        uint32_t layer_idx;
+        std::string name;
+        size_t node_id;
+    };
     
     size_t input(const std::vector<size_t>& shape, Precision precision = Precision::INT8);
     size_t precision_cast(size_t input, Precision target_precision);
@@ -209,9 +219,11 @@ public:
     
     size_t silu(size_t input);
     size_t gelu(size_t input);
+    size_t gelu_erf(size_t input);
     
     size_t matmul(size_t input1, size_t input2, bool pretransposed_rhs = false, ComputeBackend backend = ComputeBackend::CPU);
     size_t transpose(size_t input, ComputeBackend backend = ComputeBackend::CPU);
+    size_t transposeN(size_t input, const std::vector<size_t>& permutation, ComputeBackend backend = ComputeBackend::CPU);
     size_t reshape(size_t input, const std::vector<size_t>& new_shape);
     size_t slice(size_t input, int axis, size_t start, size_t length);
     size_t index(size_t input, size_t index_value, int dim);
@@ -225,9 +237,11 @@ public:
     size_t gather(size_t embeddings, size_t indices);
     size_t mmap_embeddings(const std::string& filename);
     size_t mmap_weights(const std::string& filename);
+    size_t load_weights(const std::string& filename); 
     void set_quantization_scale(size_t node_id, float scale);
     size_t embedding(const std::string& filename, size_t indices);
     size_t embedding(size_t embedding_tensor, size_t indices);
+    size_t bilinear_interpolation(size_t pos_embeds, size_t dst_height, size_t dst_width);
 
     size_t layernorm(size_t input, size_t weight, size_t bias, float epsilon = 1e-5f);
     size_t topk(size_t input, size_t k);
@@ -239,6 +253,7 @@ public:
     size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, size_t window_size, ComputeBackend backend = ComputeBackend::CPU);
 
     size_t conv1d_causal(size_t input, size_t weight, size_t kernel_size, size_t dilation = 1);
+    size_t conv1d_k3(size_t input, size_t weight, size_t stride);
     
     size_t sample(size_t logits, float temperature = 0.6f, float top_p = 0.95f, size_t top_k = 20);
     
@@ -252,6 +267,11 @@ public:
     void execute(const std::string& profile_file = "");
     void hard_reset();
     void soft_reset();
+
+    void register_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id);
+    void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id);
+    const std::vector<DebugNodeEntry>& get_debug_nodes() const;
+    void clear_debug_nodes();
     
     size_t add_node(OpType op_type, const std::vector<size_t>& inputs, const std::vector<size_t>& output_shape, const OpParams& params = {});
     const BufferDesc& get_output_buffer(size_t node_id) const;
@@ -265,6 +285,7 @@ private:
     size_t next_node_id_;
     std::vector<std::unique_ptr<GraphFile::MappedFile>> mapped_files_;
     std::unordered_map<std::string, size_t> weight_cache_;
+    std::vector<DebugNodeEntry> debug_nodes_;
 };
 
 

package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/kernel.h

@@ -174,6 +174,15 @@ void cactus_gelu_f16(const __fp16* input, __fp16* output, size_t num_elements);
 void cactus_gelu_int8(const int8_t* input, int8_t* output, size_t num_elements,
                       float input_scale, float output_scale);
 
+void cactus_gelu_f32_erf(const float* input, float* output, size_t num_elements);
+void cactus_gelu_f16_erf(const __fp16* input, __fp16* output, size_t num_elements);
+void cactus_gelu_int8_erf(
+    const int8_t* input,
+    int8_t* output,
+    size_t num_elements,
+    float scale_in,
+    float scale_out);
+
                       
 void cactus_attention_int8(const int8_t* queries, const int8_t* keys, const int8_t* values, int8_t* output,
                             size_t batch_size, size_t seq_len, size_t kv_seq_len, size_t num_q_heads, size_t num_kv_heads,
@@ -225,6 +234,49 @@ void cactus_conv1d_causal_depthwise_int8(
     float weight_scale,
     float output_scale);
 
+void cactus_conv1d_f32_k3(
+    const float* input,
+    const float* weight,
+    float* output,
+    size_t N,
+    size_t L,
+    size_t C_in,
+    size_t C_out,
+    size_t stride
+);
+
+void cactus_conv1d_f16_k3(
+    const __fp16* input,
+    const __fp16* weight,
+    __fp16* output,
+    size_t N,
+    size_t L,
+    size_t C_in,
+    size_t C_out,
+    size_t stride
+);
+
+void cactus_conv1d_f32_k3(
+    const float* input,
+    const float* weight,
+    float* output,
+    size_t N, size_t L,
+    size_t C_in, size_t C_out,
+    size_t stride
+);
+
+void cactus_conv1d_f16_k3(
+    const __fp16* input,
+    const __fp16* weight,
+    __fp16* output,
+    size_t N, size_t L,
+    size_t C_in, size_t C_out,
+    size_t stride
+);
+
+void cactus_bilinear_interpolation_fp32(const float* input, float* output, size_t src_height, size_t src_width, size_t embed_dim,
+                                        size_t dst_height, size_t dst_width);
+
 void cactus_sample_f32(const float* logits, uint32_t* output, size_t vocab_size,
                        float temperature, float top_p, size_t top_k, size_t random_seed);
 void cactus_sample_f16(const __fp16* logits, uint32_t* output, size_t vocab_size,

package/ios/cactus.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_ffi.h

@@ -20,7 +20,7 @@ typedef void* cactus_model_t;
 
 typedef void (*cactus_token_callback)(const char* token, uint32_t token_id, void* user_data);
 
-CACTUS_FFI_EXPORT cactus_model_t cactus_init(const char* model_path, size_t context_size);
+CACTUS_FFI_EXPORT cactus_model_t cactus_init(const char* model_path, size_t context_size, const char* corpus_dir);
 
 CACTUS_FFI_EXPORT int cactus_complete(
     cactus_model_t model,
@@ -33,6 +33,17 @@ CACTUS_FFI_EXPORT int cactus_complete(
     void* user_data
 );
 
+CACTUS_FFI_EXPORT int cactus_transcribe(
+    cactus_model_t model,
+    const char* audio_file_path,
+    const char* prompt,
+    char* response_buffer,
+    size_t buffer_size,
+    const char* options_json,
+    cactus_token_callback callback,
+    void* user_data
+);
+
 
 CACTUS_FFI_EXPORT int cactus_embed(
     cactus_model_t model,
@@ -42,6 +53,22 @@ CACTUS_FFI_EXPORT int cactus_embed(
     size_t* embedding_dim
 );
 
+CACTUS_FFI_EXPORT int cactus_image_embed(
+    cactus_model_t model,
+    const char* image_path,
+    float* embeddings_buffer,
+    size_t buffer_size,
+    size_t* embedding_dim
+);
+
+CACTUS_FFI_EXPORT int cactus_audio_embed(
+    cactus_model_t model,
+    const char* audio_path,
+    float* embeddings_buffer,
+    size_t buffer_size,
+    size_t* embedding_dim
+);
+
 CACTUS_FFI_EXPORT void cactus_reset(cactus_model_t model);
 
 CACTUS_FFI_EXPORT void cactus_stop(cactus_model_t model);