cactus-react-native 1.4.0 → 1.7.0

package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/engine.h

@@ -84,12 +84,17 @@ struct Config {
     bool use_thumbnail = true;
     uint32_t min_image_tokens = 64;
     uint32_t max_image_tokens = 256;
-        uint32_t max_num_patches = 1024;
+    uint32_t max_num_patches = 1024;
     uint32_t tile_size = 512;
     float max_pixels_tolerance = 2.0f;
     bool do_image_splitting = true;
+    bool encoder_act_gelu = false;
+    bool decoder_act_gelu = false;
+    uint32_t num_encoder_layers = 0;
+    uint32_t num_decoder_layers = 0;
+    float partial_rotary_factor = 0.0f;
 
-    enum class ModelType {QWEN = 0, GEMMA = 1, SMOL = 2, NOMIC = 3, LFM2 = 5, SIGLIP2 = 6, WHISPER = 7};
+    enum class ModelType {QWEN = 0, GEMMA = 1, NOMIC = 3, LFM2 = 5, SIGLIP2 = 6, WHISPER = 7, MOONSHINE = 8, SILERO_VAD = 9};
     ModelType model_type = ModelType::QWEN;
 
     enum class ModelVariant {DEFAULT = 0, VLM = 1, EXTRACT = 2, RAG = 3};
@@ -107,6 +112,8 @@ struct Config {
     float default_temperature = 0.6f;
     float default_top_p = 0.95f;
     size_t default_top_k = 20;
+    float default_max_tps = -1.0f;
+    float default_cloud_handoff_threshold = 0.0f;
 
     std::vector<std::string> layer_types;
     size_t conv_L_cache = 0;
@@ -131,9 +138,12 @@ struct MergeRule {
 struct ChatMessage {
     std::string role;
     std::string content;
+    std::string name;
     std::vector<std::string> images;
 };
 
+
+
 class Tokenizer {
 public:
     virtual ~Tokenizer() = default;
@@ -149,6 +159,7 @@ public:
     virtual uint32_t get_bos_token() const = 0;
     virtual uint32_t get_eos_token() const = 0;
     virtual bool has_chat_template() const { return has_chat_template_; }
+    std::string get_default_stop_sequence() const;
 
     virtual bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) = 0;
     
@@ -156,11 +167,8 @@ public:
     uint32_t get_fake_token_id() const { return fake_token_id_; }
     uint32_t get_global_img_token_id() const { return global_img_token_id_; }
 
-
-    void set_corpus_dir(const std::string& dir) { corpus_dir_ = dir; }
-
 protected:
-    enum class ModelType { UNKNOWN, QWEN, GEMMA, LFM2, SMOL, BERT, WHISPER};
+    enum class ModelType { UNKNOWN, QWEN, GEMMA, LFM2, BERT, WHISPER};
     ModelType model_type_ = ModelType::UNKNOWN;
     enum class ModelVariant { DEFAULT, VLM, EXTRACT, RAG};
     ModelVariant model_variant_ = ModelVariant::DEFAULT;
@@ -170,14 +178,12 @@ protected:
     uint32_t image_token_id_ = 396;
     uint32_t fake_token_id_ = 49189;
     uint32_t global_img_token_id_ = 49152;
-    std::string corpus_dir_;
 
     void detect_model_type(const std::string& config_path);
     std::string format_qwen_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt, const std::string& tools_json) const;
     std::string format_gemma_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt, const std::string& tools_json) const;
     std::string format_lfm2_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt, const std::string& tools_json) const;
     std::string format_lfm2_vl_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt, const std::string& tools_json) const;
-    std::string format_smol_style(const std::vector<ChatMessage>& messages, bool add_generation_prompt, const std::string& tools_json) const;
 };
 
 class BPETokenizer : public Tokenizer {
@@ -329,6 +335,8 @@ struct KVCache {
     struct LayerCache {
         std::vector<uint8_t> keys;
         std::vector<uint8_t> values;
+        std::vector<float> key_scales;   
+        std::vector<float> value_scales; 
     };
 
     std::vector<LayerCache> layer_caches;
@@ -354,13 +362,11 @@ struct KVCache {
                           const std::vector<size_t>& v_nodes, size_t seq_len,
                           size_t num_layers, size_t kv_heads, size_t head_dim);
 
-    // Update KV cache from NPU prefill outputs
-    // NPU outputs are in shape [num_tokens, num_kv_heads, head_dim]
-    // This handles transposition to cache format and sliding window
     void update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data,
                          size_t num_tokens, size_t kv_heads, size_t head_dim);
 
     bool is_empty() const { return current_seq_len == 0; }
+    bool is_int8() const { return precision == Precision::INT8; }
     void* get_key_ptr(size_t layer);
     void* get_value_ptr(size_t layer);
 
@@ -374,33 +380,44 @@ struct KVCache {
 
     CircularView get_key_view(size_t layer);
     CircularView get_value_view(size_t layer);
+
+    const int8_t* get_keys_int8(size_t layer) const;
+    const int8_t* get_values_int8(size_t layer) const;
+    const float* get_key_scales(size_t layer) const;
+    const float* get_value_scales(size_t layer) const;
 };
 
 class ToolCallConstrainer {
 public:
     enum class State {
-        START,                  // -> expect {
-        EXPECT_FC_KEY,          // -> expect "function_call"
-        EXPECT_FC_COLON,        // -> expect :
-        EXPECT_FC_OPEN_BRACE,   // -> expect {
-        EXPECT_NAME_KEY,        // -> expect "name"
-        EXPECT_NAME_COLON,      // -> expect :
-        EXPECT_NAME_VALUE,      // -> expect "<function_name>"
-        EXPECT_COMMA,           // -> expect ,
-        EXPECT_ARGS_KEY,        // -> expect "arguments"
-        EXPECT_ARGS_COLON,      // -> expect :
-        IN_ARGUMENTS,           // -> free JSON, track brace depth
-        EXPECT_INNER_CLOSE,     // -> expect } to close inner object
-        EXPECT_OUTER_CLOSE,     // -> expect } to close outer object
-        DONE,                   // complete
-
-        LFM_START,              // -> expect <|tool_call_start|>
-        LFM_EXPECT_BRACKET,     // -> expect [
-        LFM_IN_FUNC_NAME,       // -> expect function name
-        LFM_EXPECT_PAREN,       // -> expect (
-        LFM_IN_ARGUMENTS,       // -> arguments until )
-        LFM_EXPECT_BRACKET_CLOSE, // -> expect ]
-        LFM_EXPECT_END          // -> expect <|tool_call_end|>
+        DONE,                   
+
+        QWEN_START,             
+        QWEN_EXPECT_OPEN_BRACE, 
+        QWEN_EXPECT_NAME_KEY, 
+        QWEN_EXPECT_NAME_COLON,
+        QWEN_EXPECT_NAME_VALUE,
+        QWEN_EXPECT_COMMA, 
+        QWEN_EXPECT_ARGS_KEY, 
+        QWEN_EXPECT_ARGS_COLON, 
+        QWEN_IN_ARGUMENTS,  
+        QWEN_EXPECT_CLOSE_BRACE,
+        QWEN_EXPECT_END, 
+
+        LFM_START,              
+        LFM_EXPECT_BRACKET, 
+        LFM_IN_FUNC_NAME,
+        LFM_EXPECT_PAREN,
+        LFM_IN_ARGUMENTS, 
+        LFM_EXPECT_BRACKET_CLOSE, 
+        LFM_EXPECT_END,   
+
+        GEMMA_START,           
+        GEMMA_EXPECT_CALL, 
+        GEMMA_IN_FUNC_NAME, 
+        GEMMA_EXPECT_BRACE, 
+        GEMMA_IN_ARGUMENTS, 
+        GEMMA_EXPECT_END 
     };
 
     void init(Config::ModelType model_type,
@@ -417,42 +434,48 @@ public:
 
 private:
     bool active_ = false;
-    State state_ = State::START;
+    State state_ = State::QWEN_START;
     Config::ModelType model_type_ = Config::ModelType::QWEN;
     Tokenizer* tokenizer_ = nullptr;
 
     std::vector<std::string> function_names_;
     std::string generated_text_;
-    int brace_depth_ = 0;  // Track nested braces in arguments
-
-    // Pre-tokenized token sets for each grammar element
-    std::unordered_set<uint32_t> open_brace_tokens_;      // {
-    std::unordered_set<uint32_t> close_brace_tokens_;     // }
-    std::unordered_set<uint32_t> colon_tokens_;           // :
-    std::unordered_set<uint32_t> comma_tokens_;           // ,
-    std::unordered_set<uint32_t> fc_key_tokens_;          // "function_call"
-    std::unordered_set<uint32_t> name_key_tokens_;        // "name"
-    std::unordered_set<uint32_t> args_key_tokens_;        // "arguments"
-    std::unordered_set<uint32_t> quote_tokens_;           // "
-    std::unordered_set<uint32_t> backtick_tokens_;        // ` (to block markdown code fences)
-    std::unordered_set<uint32_t> response_starter_tokens_; // Common response starters to block (I, I'm, Sorry, etc.)
-    std::unordered_set<uint32_t> all_func_name_tokens_;   // All function name tokens combined
-    std::unordered_map<std::string, std::vector<uint32_t>> func_name_sequences_;  // Full token sequence per function
-
-    // LFM2-specific tokens
+    int brace_depth_ = 0;  
+
+    std::unordered_set<uint32_t> qwen_tool_call_start_tokens_; 
+    std::unordered_set<uint32_t> qwen_tool_call_end_tokens_;   
+    std::unordered_set<uint32_t> open_brace_tokens_;         
+    std::unordered_set<uint32_t> close_brace_tokens_;       
+    std::unordered_set<uint32_t> colon_tokens_;            
+    std::unordered_set<uint32_t> comma_tokens_;          
+    std::unordered_set<uint32_t> name_key_tokens_;           
+    std::unordered_set<uint32_t> args_key_tokens_;         
+    std::unordered_set<uint32_t> quote_tokens_;            
+    std::unordered_set<uint32_t> backtick_tokens_;   
+    std::unordered_set<uint32_t> all_func_name_tokens_;
+    std::unordered_map<std::string, std::vector<uint32_t>> func_name_sequences_;  
+
     std::unordered_set<uint32_t> tool_start_tokens_;
     std::unordered_set<uint32_t> tool_end_tokens_;
-    std::unordered_set<uint32_t> bracket_open_tokens_;    // [
-    std::unordered_set<uint32_t> bracket_close_tokens_;   // ]
-    std::unordered_set<uint32_t> paren_open_tokens_;      // (
-    std::unordered_set<uint32_t> paren_close_tokens_;     // )
-    std::unordered_set<uint32_t> equals_tokens_;          // =
+    std::unordered_set<uint32_t> bracket_open_tokens_;   
+    std::unordered_set<uint32_t> bracket_close_tokens_;  
+    std::unordered_set<uint32_t> paren_open_tokens_;     
+    std::unordered_set<uint32_t> paren_close_tokens_;   
+    std::unordered_set<uint32_t> equals_tokens_;        
+
+    std::unordered_set<uint32_t> gemma_call_start_tokens_;    
+    std::unordered_set<uint32_t> gemma_call_end_tokens_;       
+    std::unordered_set<uint32_t> gemma_response_start_tokens_; 
+    std::unordered_set<uint32_t> gemma_call_prefix_tokens_;    
+    std::unordered_set<uint32_t> escape_tokens_;              
 
     std::unordered_map<uint32_t, float> current_bias_;
 
     void compute_bias();
     void tokenize_grammar_elements();
     void add_tokens_for_string(const std::string& str, std::unordered_set<uint32_t>& token_set);
+    void tokenize_function_names(bool quote_names);
+    void init_common_tokens();
 };
 
 class Model {
@@ -477,22 +500,22 @@ public:
               const std::string& system_prompt = "", bool do_warmup = true);
 
     virtual uint32_t decode(const std::vector<uint32_t>& tokens, float temperature = -1.0f, float top_p = -1.0f,
-                      size_t top_k = 0, const std::string& profile_file = "");
+                      size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr);
 
     virtual void prefill(const std::vector<uint32_t>& tokens, size_t chunk_size = 256, const std::string& profile_file = "");
 
     virtual uint32_t decode_with_images(const std::vector<uint32_t>& tokens, const std::vector<std::string>& image_paths,
                                           float temperature = -1.0f, float top_p = -1.0f,
-                                          size_t top_k = 0, const std::string& profile_file = "");
+                                          size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr);
 
-    virtual uint32_t decode_with_audio(const std::vector<uint32_t>& tokens, const std::vector<float>& mel_bins, float temperature = 0.0f, float top_p = 0.0f,
-                      size_t top_k = 0, const std::string& profile_file = "");
+    virtual uint32_t decode_with_audio(const std::vector<uint32_t>& tokens, const std::vector<float>& audio_features, float temperature = 0.0f, float top_p = 0.0f,
+                      size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr);
 
     std::vector<float> get_embeddings(const std::vector<uint32_t>& tokens, bool pooled = true, bool normalize = false, const std::string& profile_file = "");
     
     virtual std::vector<float> get_image_embeddings(const std::string& image_path);
     
-    virtual std::vector<float> get_audio_embeddings(const std::vector<float>& mel_bins);
+    virtual std::vector<float> get_audio_embeddings(const std::vector<float>& audio_features);
 
     virtual void reset_cache() { kv_cache_.reset(); }
 
@@ -515,7 +538,7 @@ public:
 protected:
     virtual size_t forward(const std::vector<uint32_t>& tokens, bool use_cache = false) = 0;
     
-    virtual size_t forward(const std::vector<float>& mel_bins, const std::vector<uint32_t>& tokens, bool use_cache = false);
+    virtual size_t forward(const std::vector<float>& audio_features, const std::vector<uint32_t>& tokens, bool use_cache = false);
     
     virtual void load_weights_to_graph(CactusGraph* gb) = 0;
     
@@ -627,6 +650,7 @@ public:
 private:
     Config config_;
 
+    std::pair<int64_t, int64_t> compute_pixel_limits() const;
     std::vector<unsigned char> convert_to_rgb(const unsigned char* img_data, int width, int height, int channels);
     std::pair<int, int> smart_resize(int height, int width);
     bool is_image_too_large(int height, int width);
@@ -683,5 +707,102 @@ private:
     size_t num_mel_filters_;
 };
 
+namespace index {
+    constexpr uint32_t MAGIC = 0x43414354;
+    constexpr uint32_t VERSION = 1;
+
+    struct Document {
+        int id;
+        std::vector<float> embedding;
+        std::string content;
+        std::string metadata;
+    };
+
+    struct QueryResult {
+        int doc_id;
+        float score;
+    };
+
+    struct QueryOptions {
+        size_t top_k = 10;
+        float score_threshold = -1.0f;
+    };
+
+    class Index {
+        public:
+            Index(const std::string& index_path, const std::string& data_path, size_t embedding_dim);
+            ~Index();
+
+            Index(const Index&) = delete;
+            Index& operator=(const Index&) = delete;
+            Index(Index&&) = delete;
+            Index& operator=(Index&&) = delete;
+
+            void add_documents(const std::vector<Document>& documents);
+            void delete_documents(const std::vector<int>& doc_ids);
+            std::vector<Document> get_documents(const std::vector<int>& doc_ids);
+            std::vector<std::vector<QueryResult>> query(const std::vector<std::vector<float>>& embeddings, const QueryOptions& options);
+            void compact();
+
+        private:
+            struct IndexHeader {
+                uint32_t magic;
+                uint32_t version;
+                uint32_t embedding_dim;
+                uint32_t num_documents;
+            };
+
+            struct IndexEntry {
+                int32_t doc_id;
+                uint64_t data_offset;
+                uint8_t flags; // bit 0: tombstone
+
+                const __fp16* embedding() const {
+                    return reinterpret_cast<const __fp16*>(this + 1);
+                }
+
+                static size_t size(size_t embedding_dim) {
+                    return sizeof(IndexEntry) + embedding_dim * sizeof(__fp16);
+                }
+            };
+
+            struct DataHeader {
+                uint32_t magic;
+                uint32_t version;
+            };
+
+            struct DataEntry {
+                uint16_t content_len;
+                uint16_t metadata_len;
+
+                const char* content() const {
+                    return reinterpret_cast<const char*>(this + 1);
+                }
+
+                const char* metadata() const {
+                    return content() + content_len;
+                }
+            };
+
+            void parse_index_header();
+            void parse_data_header();
+            void build_doc_id_map();
+            void validate_documents(const std::vector<Document>& documents);
+            void validate_doc_ids(const std::vector<int>& doc_ids);
+            ssize_t write_full(int fd, const void* buf, size_t count);
+
+            std::unordered_map<int, uint32_t> doc_id_map_;
+
+            std::string index_path_, data_path_;
+            size_t embedding_dim_;
+            size_t index_entry_size_;
+            uint32_t num_documents_;
+
+            int index_fd_, data_fd_;
+            void *mapped_index_, *mapped_data_;
+            size_t index_file_size_, data_file_size_;
+    };
+} // namespace index
+
+}
 }
-}