cactus-react-native 1.4.0 → 1.7.0

package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/kernel_utils.h

@@ -2,6 +2,14 @@
 #define KERNEL_UTILS_H
 
 #include <arm_neon.h>
+#if defined(__APPLE__)
+#include <TargetConditionals.h>
+#include <sys/sysctl.h>
+#endif
+#if defined(__ANDROID__)
+#include <sys/auxv.h>
+#include <asm/hwcap.h>
+#endif
 #include <algorithm>
 #include <cmath>
 #include <thread>
@@ -19,166 +27,304 @@
 #include <cstdio>
 
 constexpr size_t NEON_VECTOR_SIZE = 16;
+constexpr size_t STREAMING_STORE_THRESHOLD = 32768;
 
-inline int8_t clamp_to_int8(float value) {
-    int32_t clamped = static_cast<int32_t>(roundf(value));
-    return static_cast<int8_t>(std::max(-128, std::min(127, clamped)));
+inline void stream_store_f16x8(__fp16* dst, float16x8_t val) {
+#if defined(__aarch64__)
+    float16x4_t lo = vget_low_f16(val);
+    float16x4_t hi = vget_high_f16(val);
+    __asm__ __volatile__(
+        "stnp %d0, %d1, [%2]"
+        :
+        : "w"(lo), "w"(hi), "r"(dst)
+        : "memory"
+    );
+#else
+    vst1q_f16(dst, val);
+#endif
 }
 
-inline int8_t clamp_to_int8(int32_t value) {
-    return static_cast<int8_t>(std::max(-128, std::min(127, value)));
-}
 
-#if defined(__ARM_FEATURE_DOTPROD)
-inline int32x4_t accum_i8mm(int32x4_t acc, int8x16_t a, int8x16_t b) {
-    return vdotq_s32(acc, a, b);
-}
-#else
-inline int32x4_t accum_i8mm(int32x4_t acc, int8x16_t a, int8x16_t b) {
-    int16x8_t prod_low = vmull_s8(vget_low_s8(a), vget_low_s8(b));
-    int32x4_t acc_high = vpaddlq_s16(vmull_s8(vget_high_s8(a), vget_high_s8(b)));
-    return vaddq_s32(vaddq_s32(acc, vpaddlq_s16(prod_low)), acc_high);
-}
-#endif
+inline float32x4_t fast_exp_f32x4(float32x4_t x) {
+    const float32x4_t log2e = vdupq_n_f32(1.4426950408889634f);
+    const float32x4_t ln2 = vdupq_n_f32(0.6931471805599453f);
+
+    const float32x4_t c0 = vdupq_n_f32(1.0f);
+    const float32x4_t c1 = vdupq_n_f32(0.6931471805599453f); 
+    const float32x4_t c2 = vdupq_n_f32(0.2402265069591007f);  
+    const float32x4_t c3 = vdupq_n_f32(0.05550410866482158f);
+    const float32x4_t c4 = vdupq_n_f32(0.009618129842071803f); 
+
+    x = vmaxq_f32(x, vdupq_n_f32(-87.0f));
+    x = vminq_f32(x, vdupq_n_f32(87.0f));
+
+    float32x4_t z = vmulq_f32(x, log2e);
+
+    int32x4_t zi = vcvtq_s32_f32(z);
+    float32x4_t zf = vsubq_f32(z, vcvtq_f32_s32(zi));
+
+    uint32x4_t neg_mask = vcltq_f32(zf, vdupq_n_f32(0.0f));
+    zi = vsubq_s32(zi, vandq_s32(vreinterpretq_s32_u32(neg_mask), vdupq_n_s32(1)));
+    zf = vaddq_f32(zf, vreinterpretq_f32_u32(vandq_u32(neg_mask, vreinterpretq_u32_f32(vdupq_n_f32(1.0f)))));
+
+    float32x4_t zf_ln2 = vmulq_f32(zf, ln2);
+    float32x4_t p = c4;
+    p = vfmaq_f32(c3, p, zf_ln2);
+    p = vfmaq_f32(c2, p, zf_ln2);
+    p = vfmaq_f32(c1, p, zf_ln2);
+    p = vfmaq_f32(c0, p, zf_ln2);
+
+    int32x4_t exp_bits = vshlq_n_s32(vaddq_s32(zi, vdupq_n_s32(127)), 23);
+    float32x4_t scale = vreinterpretq_f32_s32(exp_bits);
 
-inline float16x8_t accum_f16_dot(float16x8_t acc, float16x8_t a_low, float16x8_t a_high, 
-                                 float16x8_t b_low, float16x8_t b_high) {
-    acc = vfmaq_f16(acc, a_low, b_low);
-    return vfmaq_f16(acc, a_high, b_high);
+    return vmulq_f32(p, scale);
 }
 
-inline float32x4_t accum_f32_dot(float32x4_t acc, float32x4_t a_low, float32x4_t a_high, 
-                                  float32x4_t b_low, float32x4_t b_high) {
-    acc = vfmaq_f32(acc, a_low, b_low);
-    return vfmaq_f32(acc, a_high, b_high);
+inline float32x4_t fast_tanh_f32x4(float32x4_t x) {
+    const float32x4_t one = vdupq_n_f32(1.0f);
+    const float32x4_t neg_one = vdupq_n_f32(-1.0f);
+
+    uint32x4_t pos_sat = vcgtq_f32(x, vdupq_n_f32(4.5f));
+    uint32x4_t neg_sat = vcltq_f32(x, vdupq_n_f32(-4.5f));
+
+    const float32x4_t c27 = vdupq_n_f32(27.0f);
+    const float32x4_t c9 = vdupq_n_f32(9.0f);
+
+    float32x4_t x2 = vmulq_f32(x, x);
+    float32x4_t num = vaddq_f32(c27, x2);   
+    float32x4_t den = vfmaq_f32(c27, c9, x2);  
+
+    float32x4_t result = vmulq_f32(x, vdivq_f32(num, den));
+
+    result = vbslq_f32(pos_sat, one, result);
+    result = vbslq_f32(neg_sat, neg_one, result);
+
+    return result;
 }
 
 namespace CactusThreading {
-    
+
     class ThreadPool {
     private:
+        static constexpr size_t MAX_WORKERS = 16;
+
         std::vector<std::thread> workers;
-        std::queue<std::function<void()>> tasks;
-        std::mutex queue_mutex;
-        std::condition_variable condition;
-        std::atomic<bool> stop{false};
-        std::atomic<size_t> active_workers{0};
-        std::condition_variable finish_condition;
-        
+        std::deque<std::function<void()>> tasks;
+
+        std::mutex mutex;
+        std::condition_variable work_available;
+        std::condition_variable work_done;
+
+        bool stop{false};
+        std::atomic<size_t> pending_tasks{0};
+        size_t num_workers_;
+
         void worker_thread() {
             while (true) {
                 std::function<void()> task;
                 {
-                    std::unique_lock<std::mutex> lock(queue_mutex);
-                    condition.wait(lock, [this] { return stop || !tasks.empty(); });
-                    
-                    if (stop && tasks.empty()) return;
-                    
+                    std::unique_lock<std::mutex> lock(mutex);
+                    work_available.wait(lock, [this] {
+                        return stop || !tasks.empty();
+                    });
+
+                    if (stop && tasks.empty()) {
+                        return;
+                    }
+
                     task = std::move(tasks.front());
-                    tasks.pop();
-                    active_workers++;
+                    tasks.pop_front();
                 }
-                
+
                 task();
-                
-                active_workers--;
-                finish_condition.notify_all();
+
+                if (pending_tasks.fetch_sub(1, std::memory_order_acq_rel) == 1) {
+                    std::lock_guard<std::mutex> lock(mutex);
+                    work_done.notify_one();
+                }
             }
         }
-        
+
     public:
-        explicit ThreadPool(size_t num_threads = std::thread::hardware_concurrency()) {
-            workers.reserve(num_threads);
-            for (size_t i = 0; i < num_threads; ++i) {
+        explicit ThreadPool(size_t num_threads = std::thread::hardware_concurrency())
+            : stop(false), pending_tasks(0) {
+            num_workers_ = std::min(num_threads, MAX_WORKERS);
+            if (num_workers_ == 0) num_workers_ = 1;
+            workers.reserve(num_workers_);
+            for (size_t i = 0; i < num_workers_; ++i) {
                 workers.emplace_back(&ThreadPool::worker_thread, this);
             }
         }
-        
+
         ~ThreadPool() {
             {
-                std::unique_lock<std::mutex> lock(queue_mutex);
+                std::lock_guard<std::mutex> lock(mutex);
                 stop = true;
             }
-            condition.notify_all();
+            work_available.notify_all();
             for (auto& worker : workers) {
-                worker.join();
+                if (worker.joinable()) {
+                    worker.join();
+                }
             }
         }
-        
+
         template<typename F>
         auto enqueue(F&& f) -> std::future<decltype(f())> {
             using return_type = decltype(f());
-            
+
             auto task = std::make_shared<std::packaged_task<return_type()>>(
                 std::forward<F>(f)
             );
-            
+
             std::future<return_type> res = task->get_future();
+
             {
-                std::unique_lock<std::mutex> lock(queue_mutex);
-                if (stop) throw std::runtime_error("enqueue on stopped ThreadPool");
-                
-                tasks.emplace([task](){ (*task)(); });
+                std::lock_guard<std::mutex> lock(mutex);
+                pending_tasks.fetch_add(1, std::memory_order_relaxed);
+                tasks.emplace_back([task](){ (*task)(); });
             }
-            condition.notify_one();
+            work_available.notify_one();
+
             return res;
         }
-        
+
+        template<typename F>
+        void enqueue_batch(size_t total_work, F task_func) {
+            if (total_work == 0) return;
+
+            const size_t num_tasks = std::min(num_workers_, total_work);
+            const size_t per_worker = total_work / num_tasks;
+            const size_t remainder = total_work % num_tasks;
+
+            {
+                std::lock_guard<std::mutex> lock(mutex);
+                pending_tasks.fetch_add(num_tasks, std::memory_order_relaxed);
+
+                for (size_t w = 0; w < num_tasks; ++w) {
+                    size_t start = w * per_worker + std::min(w, remainder);
+                    size_t end = start + per_worker + (w < remainder ? 1 : 0);
+                    tasks.emplace_back([=]() { task_func(start, end); });
+                }
+            }
+            work_available.notify_all();
+        }
+
         void wait_all() {
-            std::unique_lock<std::mutex> lock(queue_mutex);
-            finish_condition.wait(lock, [this] { 
-                return tasks.empty() && active_workers == 0; 
+            std::unique_lock<std::mutex> lock(mutex);
+            work_done.wait(lock, [this] {
+                return pending_tasks.load(std::memory_order_acquire) == 0;
             });
         }
-        
-        size_t num_workers() const { return workers.size(); }
+
+        template<typename F>
+        void enqueue_n_threads(size_t total_work, size_t num_threads, F task_func) {
+            if (total_work == 0 || num_threads == 0) return;
+
+            num_threads = std::min(num_threads, std::min(num_workers_, total_work));
+            const size_t per_thread = total_work / num_threads;
+            const size_t remainder = total_work % num_threads;
+
+            {
+                std::lock_guard<std::mutex> lock(mutex);
+                pending_tasks.fetch_add(num_threads, std::memory_order_relaxed);
+
+                for (size_t t = 0; t < num_threads; ++t) {
+                    size_t start = t * per_thread + std::min(t, remainder);
+                    size_t end = start + per_thread + (t < remainder ? 1 : 0);
+                    tasks.emplace_back([=]() { task_func(start, end); });
+                }
+            }
+            work_available.notify_all();
+        }
+
+        size_t num_workers() const { return num_workers_; }
     };
-    
+
     inline ThreadPool& get_thread_pool() {
         static ThreadPool pool;
         return pool;
     }
     
-    inline size_t get_optimal_thread_count(size_t total_work, size_t min_work_per_thread) {
-        if (total_work < min_work_per_thread) return 1;
+    struct ParallelConfig {
+        size_t min_work_gate;  
+        size_t work_per_thread; 
+
+        constexpr ParallelConfig(size_t gate, size_t per_thread)
+            : min_work_gate(gate), work_per_thread(per_thread) {}
+    };
+
+    inline size_t get_optimal_thread_count(size_t total_work, ParallelConfig config) {
+        if (total_work < config.min_work_gate) return 1;
+
         size_t pool_size = get_thread_pool().num_workers();
-        return std::min(pool_size, 
-                       std::max(static_cast<size_t>(1), total_work / min_work_per_thread));
+        size_t num_threads = (total_work + config.work_per_thread - 1) / config.work_per_thread;
+        return std::min(pool_size, std::max(static_cast<size_t>(1), num_threads));
     }
-    
+
     struct Thresholds {
+        #if defined(__ANDROID__)
+        static constexpr ParallelConfig ATTENTION{64, 32};
+        static constexpr ParallelConfig ELEMENT_WISE{5000, 2500};
+        static constexpr ParallelConfig AXIS_REDUCE{1000, 500};
+        static constexpr ParallelConfig ALL_REDUCE{10000, 5000};
+        static constexpr ParallelConfig SCALAR_BASIC{30000, 15000};
+        static constexpr ParallelConfig SCALAR_EXPENSIVE{10000, 5000};
+        #else // Apple
+        static constexpr ParallelConfig ATTENTION{32, 16};
+        static constexpr ParallelConfig ELEMENT_WISE{5000, 2500};
+        static constexpr ParallelConfig AXIS_REDUCE{1000, 500};
+        static constexpr ParallelConfig ALL_REDUCE{10000, 5000};
+        static constexpr ParallelConfig SCALAR_BASIC{5000, 2500};
+        static constexpr ParallelConfig SCALAR_EXPENSIVE{2500, 1250};
+        #endif
+    };
 
+    struct GemmThreading {
         #if defined(__ANDROID__)
-        static constexpr size_t ELEMENT_WISE = 5000;
-        static constexpr size_t AXIS_REDUCE = 1000;
-        static constexpr size_t ALL_REDUCE = 10000;
-        static constexpr size_t SCALAR_BASIC = 30000;
-        static constexpr size_t SCALAR_EXPENSIVE = 10000;
-        static constexpr size_t ATTENTION = 512;
-        static constexpr size_t GEMM_TILED = 20000; 
-        static constexpr size_t GEMM_SMALL = 64 * 64 * 64;
-        static constexpr size_t GEMM_MEDIUM = 256 * 256 * 256;
-        static constexpr size_t GEMM_TILE_M = 64;
-        static constexpr size_t GEMM_TILE_N = 64;
-        static constexpr size_t GEMM_TILE_M_SMALL = 32;
-        static constexpr size_t GEMM_TILE_N_SMALL = 32;
-        #else // iOS
-        static constexpr size_t ELEMENT_WISE = 5000;
-        static constexpr size_t AXIS_REDUCE = 1000;
-        static constexpr size_t ALL_REDUCE = 10000;
-        static constexpr size_t SCALAR_BASIC = 5000;
-        static constexpr size_t SCALAR_EXPENSIVE = 2500;
-        static constexpr size_t ATTENTION = 4;
-        static constexpr size_t GEMM_TILED = 4;  
-        static constexpr size_t GEMM_SMALL = 64 * 64 * 64;
-        static constexpr size_t GEMM_MEDIUM = 256 * 256 * 256;
-        static constexpr size_t GEMM_TILE_M = 64;
-        static constexpr size_t GEMM_TILE_N = 64;
-        static constexpr size_t GEMM_TILE_M_SMALL = 32;
-        static constexpr size_t GEMM_TILE_N_SMALL = 32;
+        static size_t get_num_threads(size_t M, size_t pool_size) {
+            if (M <= 1) return 1;
+            return pool_size;
+        }
+        static size_t get_gemv_threads(size_t /*N_blocks*/, size_t /*pool_size*/) {
+            return 1; 
+        }
+        #elif defined(__APPLE__) && TARGET_OS_IPHONE
+        static constexpr size_t GEMV_MIN_N_BLOCKS = 512; 
+        static size_t get_num_threads(size_t M, size_t pool_size) {
+            if (M <= 1) return std::min(pool_size, static_cast<size_t>(2));
+            return pool_size;
+        }
+        static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) {
+            if (N_blocks < GEMV_MIN_N_BLOCKS) return 1;
+            return std::min(pool_size, static_cast<size_t>(2));
+        }
+        #else 
+        static constexpr size_t GEMV_MIN_N_BLOCKS = 256;  
+        static size_t get_num_threads(size_t M, size_t pool_size) {
+            if (M <= 1) return std::min(pool_size, static_cast<size_t>(4));
+            return pool_size;
+        }
+        static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) {
+            if (N_blocks < GEMV_MIN_N_BLOCKS) return 1;
+            if (N_blocks < 512) return std::min(pool_size, static_cast<size_t>(2));
+            return std::min(pool_size, static_cast<size_t>(4));
+        }
         #endif
-        static constexpr size_t L2_CACHE_SIZE = 256 * 1024;
     };
+
+    inline size_t& get_gemm_thread_override() {
+        static size_t override_threads = 0; 
+        return override_threads;
+    }
+
+    inline void set_gemm_threads(size_t num_threads) {
+        get_gemm_thread_override() = num_threads;
+    }
+
+    inline void reset_gemm_threads() {
+        get_gemm_thread_override() = 0;
+    }
     
     class TaskHandle {
     private:
@@ -225,10 +371,10 @@ namespace CactusThreading {
     };
     
     template<typename WorkFunc>
-    TaskHandle parallel_for(size_t total_work, size_t threshold, WorkFunc work_func, bool wait = true) {
-        const size_t num_threads = get_optimal_thread_count(total_work, threshold);
-        TaskHandle handle(!wait);  
-        
+    TaskHandle parallel_for(size_t total_work, ParallelConfig config, WorkFunc work_func, bool wait = true) {
+        const size_t num_threads = get_optimal_thread_count(total_work, config);
+        TaskHandle handle(!wait);
+
         if (num_threads == 1) {
             if (wait) {
                 work_func(0, total_work);
@@ -240,10 +386,10 @@ namespace CactusThreading {
             }));
             return handle;
         }
-        
+
         auto& pool = get_thread_pool();
         const size_t work_per_thread = total_work / num_threads;
-        
+
         for (size_t t = 0; t < num_threads; ++t) {
             handle.add_future(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() {
                 const size_t start_idx = t * work_per_thread;
@@ -251,17 +397,17 @@ namespace CactusThreading {
                 work_func(start_idx, end_idx);
             }));
         }
-        
+
         if (wait) {
             handle.wait();
         }
         return handle;
     }
-    
+
     template<typename WorkFunc>
-    void parallel_for_2d(size_t outer_size, size_t inner_size, size_t threshold, WorkFunc work_func) {
+    void parallel_for_2d(size_t outer_size, size_t inner_size, ParallelConfig config, WorkFunc work_func) {
         const size_t total_work = outer_size * inner_size;
-        parallel_for(total_work, threshold, [&](size_t start_idx, size_t end_idx) {
+        parallel_for(total_work, config, [&](size_t start_idx, size_t end_idx) {
             for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) {
                 const size_t outer = work_idx / inner_size;
                 const size_t inner = work_idx % inner_size;
@@ -269,11 +415,11 @@ namespace CactusThreading {
             }
         });
     }
-    
+
     template<typename WorkFunc, typename ResultType, typename CombineFunc>
-    ResultType parallel_reduce(size_t total_work, size_t threshold, 
+    ResultType parallel_reduce(size_t total_work, ParallelConfig config,
                               WorkFunc work_func, ResultType init_value, CombineFunc combine_func) {
-        const size_t num_threads = get_optimal_thread_count(total_work, threshold);
+        const size_t num_threads = get_optimal_thread_count(total_work, config);
         
         if (num_threads == 1) {
             return work_func(0, total_work);
@@ -298,46 +444,25 @@ namespace CactusThreading {
         }
         return result;
     }
-    
-    inline size_t compute_gemm_parallelism(size_t M, size_t K, size_t N, size_t element_size) {
-        size_t total_ops = M * K * N;
-        
-        if (total_ops < Thresholds::GEMM_SMALL) return 1;
-        
-        if (total_ops < Thresholds::GEMM_MEDIUM) {
-            return std::min(static_cast<size_t>(2), get_thread_pool().num_workers());
-        }
-        
-        size_t bytes_accessed = (M * K + K * N + M * N) * element_size;
-        size_t cache_tiles = (bytes_accessed + Thresholds::L2_CACHE_SIZE - 1) / Thresholds::L2_CACHE_SIZE;
-        
-        size_t compute_threads = std::sqrt(static_cast<double>(total_ops) / Thresholds::GEMM_SMALL);
-        size_t memory_threads = cache_tiles;
-        
-        size_t optimal = std::min(compute_threads, memory_threads);
-        return std::min(optimal, get_thread_pool().num_workers());
-    }
-    
+
     template<typename WorkFunc>
-    void parallel_for_2d_tiled(size_t rows, size_t cols, size_t tile_rows, size_t tile_cols, WorkFunc work_func) {
-        size_t num_row_tiles = (rows + tile_rows - 1) / tile_rows;
-        size_t num_col_tiles = (cols + tile_cols - 1) / tile_cols;
-        size_t total_tiles = num_row_tiles * num_col_tiles;
-
-        parallel_for(total_tiles, Thresholds::GEMM_TILED, [=](size_t start_tile, size_t end_tile) {
-            for (size_t tile_idx = start_tile; tile_idx < end_tile; ++tile_idx) {
-                size_t tile_row = tile_idx / num_col_tiles;
-                size_t tile_col = tile_idx % num_col_tiles;
-                
-                size_t row_start = tile_row * tile_rows;
-                size_t row_end = std::min(row_start + tile_rows, rows);
-                size_t col_start = tile_col * tile_cols;
-                size_t col_end = std::min(col_start + tile_cols, cols);
-                
-                work_func(row_start, row_end, col_start, col_end);
-            }
-        });
+    void parallel_gemm_tiles(size_t M, size_t total_tiles, WorkFunc work_func) {
+        auto& pool = get_thread_pool();
+
+        size_t override = get_gemm_thread_override();
+        size_t num_threads = (override > 0) ? override : GemmThreading::get_num_threads(M, pool.num_workers());
+        num_threads = std::min(num_threads, total_tiles);
+
+        if (num_threads <= 1) {
+            work_func(0, total_tiles);
+            return;
+        }
+
+        pool.enqueue_n_threads(total_tiles, num_threads, work_func);
+        pool.wait_all();
     }
+
 }
 
+
 #endif // KERNEL_UTILS_H 

package/ios/cactus.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus.h

@@ -7,7 +7,6 @@
 #include "engine/engine.h"
 #include "models/model.h"
 #include "ffi/cactus_ffi.h"
-#include "ffi/cactus_telemetry.h"
 #include "npu/npu.h"
 
 #endif // CACTUS_H