cactus-react-native 1.4.0 → 1.5.0

package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/graph.h

@@ -11,6 +11,7 @@
 #include <mutex>
 #include <sstream>
 #include <iostream>
+#include <arm_neon.h>
 
 namespace cactus {
 
@@ -96,9 +97,10 @@ namespace GraphFile {
 }
 
 enum class Precision {
-    INT8, 
+    INT8,
     FP16,
-    FP32
+    FP32,
+    INT4 
 };
 
 enum class ComputeBackend {
@@ -112,7 +114,7 @@ enum class OpType {
     MATMUL, TRANSPOSE, RESHAPE, SLICE, GATHER, EMBEDDING,
     BILINEAR_INTERPOLATION,
     SUM, MEAN, VARIANCE, MIN, MAX,
-    RMS_NORM, ROPE, SOFTMAX, ATTENTION, CONV1D_CAUSAL, CONV1D_K3,
+    RMS_NORM, ROPE, SOFTMAX, ATTENTION, ATTENTION_INT8_HYBRID, CONV1D_CAUSAL, CONV1D_K3,
     SCALAR_ADD, SCALAR_SUBTRACT, SCALAR_MULTIPLY, SCALAR_DIVIDE, SCALAR_EXP, SCALAR_SQRT, SCALAR_COS, SCALAR_SIN,
     SILU, GELU, GELU_ERF,
     SAMPLE, CONCAT,
@@ -122,27 +124,38 @@ enum class OpType {
 };
 
 struct PrecisionTraits {
+    // Returns in-memory element size (INT4 unpacks to INT8, so returns 1)
     static constexpr size_t size_of(Precision prec) {
         switch (prec) {
             case Precision::INT8: return 1;
             case Precision::FP16: return 2;
             case Precision::FP32: return 4;
+            case Precision::INT4: return 1; 
         }
         return 1;
     }
-    
+
+    static constexpr size_t packed_size_of(Precision prec, size_t count) {
+        switch (prec) {
+            case Precision::INT4: return (count + 1) / 2;  
+            default: return count * size_of(prec);
+        }
+    }
+
     static constexpr bool is_integer(Precision prec) {
         switch (prec) {
             case Precision::INT8: return true;
+            case Precision::INT4: return true;
             case Precision::FP16: return false;
             case Precision::FP32: return false;
         }
         return true;
     }
-    
+
     static constexpr bool is_floating_point(Precision prec) {
         switch (prec) {
             case Precision::INT8: return false;
+            case Precision::INT4: return false;
             case Precision::FP16: return true;
             case Precision::FP32: return true;
         }
@@ -153,8 +166,6 @@ struct PrecisionTraits {
 namespace Quantization {
     void int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f);
     void fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f);
-    void dynamic_quantize_fp32_to_int8(const float* src, int8_t* dst, size_t count, 
-                                       float* computed_scale);
     void fp16_to_fp32(const __fp16* src, float* dst, size_t count);
     void fp32_to_fp16(const float* src, __fp16* dst, size_t count);
     void int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f);
@@ -188,10 +199,17 @@ struct BufferDesc {
     void* external_data;
     char* pooled_data;
     Precision precision;
-    float quantization_scale;
+
+    size_t group_size = 0;
+    size_t num_groups = 0;
+    void* scales_data = nullptr;
+    std::unique_ptr<char[]> owned_scales;
+
+    const void* packed_int4_data = nullptr;  
+    size_t packed_int4_size = 0; 
 
     BufferDesc();
-    BufferDesc(const std::vector<size_t>& s, Precision prec = Precision::INT8, float scale = 1.0f);
+    BufferDesc(const std::vector<size_t>& s, Precision prec = Precision::INT8);
     ~BufferDesc();
 
     BufferDesc(BufferDesc&& other) noexcept;
@@ -209,6 +227,28 @@ struct BufferDesc {
     template<typename T>
     const T* data_as() const { return static_cast<const T*>(get_data()); }
 
+    const __fp16* scales_as_fp16() const {
+        return reinterpret_cast<const __fp16*>(scales_data);
+    }
+    bool is_grouped_int8() const {
+        return precision == Precision::INT8 && group_size > 0;
+    }
+    bool is_packed_int4() const {
+        return packed_int4_data != nullptr && packed_int4_size > 0;
+    }
+    const uint8_t* packed_int4_as_uint8() const {
+        return reinterpret_cast<const uint8_t*>(packed_int4_data);
+    }
+    void set_grouped_scales(size_t gs, size_t ng, void* scales_ptr) {
+        group_size = gs;
+        num_groups = ng;
+        scales_data = scales_ptr;
+    }
+    void set_packed_int4(const void* packed_data, size_t packed_size) {
+        packed_int4_data = packed_data;
+        packed_int4_size = packed_size;
+    }
+
     void allocate();
     void allocate_from_pool(BufferPool& pool);
     void release_to_pool(BufferPool& pool);
@@ -247,6 +287,14 @@ struct OpParams {
 
     std::vector<float> bias_values;
     std::vector<uint32_t> bias_indices;
+
+    const int8_t* cached_keys_int8 = nullptr;
+    const int8_t* cached_values_int8 = nullptr;
+    const float* cached_k_scales = nullptr;
+    const float* cached_v_scales = nullptr;
+    size_t cache_seq_len = 0;
+    size_t num_kv_heads = 0;
+    size_t head_dim = 0;
 };
 
 struct GraphNode {
@@ -326,7 +374,7 @@ public:
     size_t precision_cast(size_t input, Precision target_precision);
     
     size_t add(size_t input1, size_t input2);
-    size_t add_clipped(size_t input1, size_t input2);  // For FP16 residual connections (Gemma)
+    size_t add_clipped(size_t input1, size_t input2);  
     size_t subtract(size_t input1, size_t input2);
     size_t multiply(size_t input1, size_t input2);
     size_t divide(size_t input1, size_t input2);
@@ -361,8 +409,12 @@ public:
     size_t gather(size_t embeddings, size_t indices);
     size_t mmap_embeddings(const std::string& filename);
     size_t mmap_weights(const std::string& filename);
-    size_t load_weights(const std::string& filename); 
-    void set_quantization_scale(size_t node_id, float scale);
+    size_t load_weights(const std::string& filename);
+    void set_grouped_scales(size_t node_id, size_t group_size, size_t num_groups, void* scales_ptr);
+
+    void release_weight_pages(size_t node_id);
+    void prefetch_weight_pages(size_t node_id);
+    void release_all_weight_pages();
     size_t embedding(const std::string& filename, size_t indices);
     size_t embedding(size_t embedding_tensor, size_t indices);
     size_t bilinear_interpolation(size_t pos_embeds, size_t dst_height, size_t dst_width);
@@ -376,6 +428,11 @@ public:
     size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, ComputeBackend backend = ComputeBackend::CPU);
     size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, size_t window_size, ComputeBackend backend = ComputeBackend::CPU);
 
+    size_t attention_int8_hybrid(size_t query, size_t key_new, size_t value_new, float scale, size_t position_offset,
+                                 const int8_t* cached_keys, const int8_t* cached_values,
+                                 const float* k_scales, const float* v_scales,
+                                 size_t cache_len, size_t num_kv_heads, size_t head_dim);
+
     size_t conv1d_causal(size_t input, size_t weight, size_t kernel_size, size_t dilation = 1);
     size_t conv1d_k3(size_t input, size_t weight, size_t stride);
     
@@ -392,6 +449,8 @@ public:
     void execute(const std::string& profile_file = "");
     void hard_reset();
     void soft_reset();
+    void soft_reset_keep_pool();
+    void set_prefill_mode(bool enabled) { prefill_mode_ = enabled; }
 
     void register_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id);
     void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id);
@@ -410,8 +469,10 @@ private:
     size_t next_node_id_;
     std::vector<std::unique_ptr<GraphFile::MappedFile>> mapped_files_;
     std::unordered_map<std::string, size_t> weight_cache_;
+    std::unordered_map<size_t, size_t> node_to_mapped_file_;
     std::vector<DebugNodeEntry> debug_nodes_;
     BufferPool buffer_pool_;
+    bool prefill_mode_ = false;
 };
 
 
@@ -430,25 +491,36 @@ namespace GraphFile {
     public:
         MappedFile(const std::string& filename);
         ~MappedFile();
-        
+
         MappedFile(const MappedFile&) = delete;
         MappedFile& operator=(const MappedFile&) = delete;
         MappedFile(MappedFile&& other) noexcept;
         MappedFile& operator=(MappedFile&& other) noexcept;
-        
+
         const std::vector<size_t>& shape() const;
         Precision precision() const;
+        Precision effective_precision() const {
+            return is_int4_ ? Precision::INT8 : precision_;
+        }
         size_t byte_size() const;
-        float quantization_scale() const;
-        
+
+        size_t group_size() const { return group_size_; }
+        size_t num_groups() const { return num_groups_; }
+        const void* scales_data() const;
+        const void* raw_packed_data() const;  // Get raw mmap'd data without unpacking (for INT4)
+        bool is_int4() const { return is_int4_; }
+
         void* data();
         const void* data() const;
-        
+
         template<typename T>
         const T* typed_data() const;
-        
+
         LoadedNode load_into_graph(CactusGraph& graph) const;
-        
+
+        void release_pages();
+        void prefetch_pages();
+
     private:
         int fd_;
         void* mapped_data_;
@@ -456,10 +528,19 @@ namespace GraphFile {
         std::vector<size_t> shape_;
         Precision precision_;
         size_t byte_size_;
-        float quantization_scale_;
+        size_t group_size_ = 0;
+        size_t num_groups_ = 0;
+        size_t scales_offset_ = 0;
+        size_t scales_bytes_ = 0;
+        uint32_t version_ = 1;
+        uint32_t alignment_ = 32;
+        bool is_int4_ = false;
+        mutable std::unique_ptr<int8_t[]> unpacked_int4_data_;
         void parse_header();
+        void apply_madvise_hints();
+        void unpack_int4_if_needed() const;
     };
-    
+
     MappedFile mmap_load(const std::string& filename);
 }
 

package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/kernel.h

@@ -15,12 +15,7 @@ enum class ScalarOpType {
     SIN
 };
 
-
-void cactus_add_int8(const int8_t* a, const int8_t* b, int8_t* output, size_t num_elements);
-void cactus_subtract_int8(const int8_t* a, const int8_t* b, int8_t* output, size_t num_elements);
-void cactus_multiply_int8(const int8_t* a, const int8_t* b, int8_t* output, size_t num_elements);
-void cactus_divide_int8(const int8_t* a, const int8_t* b, int8_t* output, size_t num_elements);
-
+constexpr size_t KV_QUANT_GROUP_SIZE = 128;
 
 void cactus_add_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements);
 void cactus_add_f16_clipped(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements);
@@ -28,27 +23,6 @@ void cactus_subtract_f16(const __fp16* a, const __fp16* b, __fp16* output, size_
 void cactus_multiply_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements);
 void cactus_divide_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements);
 
-
-void cactus_add_f32(const float* a, const float* b, float* output, size_t num_elements);
-void cactus_subtract_f32(const float* a, const float* b, float* output, size_t num_elements);
-void cactus_multiply_f32(const float* a, const float* b, float* output, size_t num_elements);
-void cactus_divide_f32(const float* a, const float* b, float* output, size_t num_elements);
-
-
-void cactus_add_broadcast_int8(const int8_t* a, const int8_t* b, int8_t* output,
-                               const size_t* a_strides, const size_t* b_strides,
-                               const size_t* output_shape, size_t ndim);
-void cactus_subtract_broadcast_int8(const int8_t* a, const int8_t* b, int8_t* output,
-                                    const size_t* a_strides, const size_t* b_strides,
-                                    const size_t* output_shape, size_t ndim);
-void cactus_multiply_broadcast_int8(const int8_t* a, const int8_t* b, int8_t* output,
-                                    const size_t* a_strides, const size_t* b_strides,
-                                    const size_t* output_shape, size_t ndim);
-void cactus_divide_broadcast_int8(const int8_t* a, const int8_t* b, int8_t* output,
-                                  const size_t* a_strides, const size_t* b_strides,
-                                  const size_t* output_shape, size_t ndim);
-
-
 void cactus_add_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output,
                                const size_t* a_strides, const size_t* b_strides,
                                const size_t* output_shape, size_t ndim);
@@ -62,159 +36,72 @@ void cactus_divide_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* outpu
                                  const size_t* a_strides, const size_t* b_strides,
                                  const size_t* output_shape, size_t ndim);
 
-
-void cactus_add_broadcast_f32(const float* a, const float* b, float* output,
-                               const size_t* a_strides, const size_t* b_strides,
-                               const size_t* output_shape, size_t ndim);
-void cactus_subtract_broadcast_f32(const float* a, const float* b, float* output,
-                                   const size_t* a_strides, const size_t* b_strides,
-                                   const size_t* output_shape, size_t ndim);
-void cactus_multiply_broadcast_f32(const float* a, const float* b, float* output,
-                                   const size_t* a_strides, const size_t* b_strides,
-                                   const size_t* output_shape, size_t ndim);
-void cactus_divide_broadcast_f32(const float* a, const float* b, float* output,
-                                 const size_t* a_strides, const size_t* b_strides,
-                                 const size_t* output_shape, size_t ndim);
-
-
-void cactus_scalar_op_int8(const int8_t* input, int8_t* output, size_t num_elements, float scalar_value, ScalarOpType op_type);
 void cactus_scalar_op_f16(const __fp16* input, __fp16* output, size_t num_elements, float scalar_value, ScalarOpType op_type);
-void cactus_scalar_op_f32(const float* input, float* output, size_t num_elements, float scalar_value, ScalarOpType op_type);
-
 
-void cactus_matmul_int8(const int8_t* a, const int8_t* b_transposed, int8_t* c,
-                        size_t M, size_t K, size_t N,
-                        float a_scale, float b_scale, float c_scale);
+void cactus_matmul_int8(const int8_t* A, const float* A_scales,
+                        const int8_t* B, const __fp16* B_scales,
+                        __fp16* C, size_t M, size_t K, size_t N, size_t group_size);
 
-#if defined(__ARM_FEATURE_MATMUL_INT8)
-void cactus_matmul_int8_to_int32_i8mm(const int8_t* a, const int8_t* b_transposed, int32_t* c,
-                                       size_t M, size_t K, size_t N);
-#define cactus_matmul_int8_to_int32 cactus_matmul_int8_to_int32_i8mm
-#else
-void cactus_matmul_int8_to_int32(const int8_t* a, const int8_t* b_transposed, int32_t* c,
-                                 size_t M, size_t K, size_t N);
-#endif
+void cactus_matmul_int4(const int8_t* A, const float* A_scales,
+                        const uint8_t* B_packed, const __fp16* B_scales,
+                        __fp16* C, size_t M, size_t K, size_t N, size_t group_size);
 
 void cactus_matmul_f16(const __fp16* a, const __fp16* b_transposed, __fp16* c,
                        size_t M, size_t K, size_t N);
 
-void cactus_matmul_f32(const float* a, const float* b_transposed, float* c,
-                       size_t M, size_t K, size_t N);
-
-
-void cactus_transpose_2d_int8(const int8_t* source, int8_t* destination,
-                               size_t num_rows, size_t num_cols, size_t start_row, size_t end_row);
 void cactus_transpose_2d_f16(const __fp16* source, __fp16* destination,
                              size_t num_rows, size_t num_cols, size_t start_row, size_t end_row);
-void cactus_transpose_2d_f32(const float* source, float* destination,
-                             size_t num_rows, size_t num_cols, size_t start_row, size_t end_row);
-
-void cactus_transpose_int8(const int8_t* source, int8_t* destination, const size_t* shape,
-                           const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx);
 void cactus_transpose_f16(const __fp16* source, __fp16* destination, const size_t* shape,
                           const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx);
-void cactus_transpose_f32(const float* source, float* destination, const size_t* shape,
-                          const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx);
 
-int64_t cactus_sum_all_int8(const int8_t* data, size_t num_elements);
-void cactus_sum_axis_int8(const int8_t* input, int8_t* output, size_t outer_size, size_t axis_size, size_t inner_size);
 double cactus_sum_all_f16(const __fp16* data, size_t num_elements);
-double cactus_sum_all_f32(const float* data, size_t num_elements);
-void cactus_sum_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size);
+void cactus_sum_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size);
 
-double cactus_mean_all_int8(const int8_t* data, size_t num_elements);
-void cactus_mean_axis_int8(const int8_t* input, int8_t* output, size_t outer_size, size_t axis_size, size_t inner_size);
 double cactus_mean_all_f16(const __fp16* data, size_t num_elements);
 void cactus_mean_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size);
-double cactus_mean_all_f32(const float* data, size_t num_elements);
-void cactus_mean_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size);
 
-double cactus_variance_all_int8(const int8_t* data, size_t num_elements);
-void cactus_variance_axis_int8(const int8_t* input, int8_t* output, size_t outer_size, size_t axis_size, size_t inner_size);
-double cactus_variance_all_f32(const float* data, size_t num_elements);
-void cactus_variance_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size);
+double cactus_variance_all_f16(const __fp16* data, size_t num_elements);
+void cactus_variance_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size);
 
-int64_t cactus_min_all_int8(const int8_t* data, size_t num_elements);
-void cactus_min_axis_int8(const int8_t* input, int8_t* output, size_t outer_size, size_t axis_size, size_t inner_size);
-float cactus_min_all_f32(const float* data, size_t num_elements);
-void cactus_min_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size);
+__fp16 cactus_min_all_f16(const __fp16* data, size_t num_elements);
+void cactus_min_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size);
 
-int64_t cactus_max_all_int8(const int8_t* data, size_t num_elements);
-void cactus_max_axis_int8(const int8_t* input, int8_t* output, size_t outer_size, size_t axis_size, size_t inner_size);
-float cactus_max_all_f32(const float* data, size_t num_elements);
-void cactus_max_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size);
+__fp16 cactus_max_all_f16(const __fp16* data, size_t num_elements);
+void cactus_max_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size);
 
 void cactus_rms_norm_f16(const __fp16* input, const __fp16* weight, __fp16* output,
                           size_t batch_size, size_t dims, float eps);
-                          
-void cactus_rms_norm_f32(const float* input, const float* weight, float* output,
-                          size_t batch_size, size_t dims, float eps);
-
-void cactus_rms_norm_i8_f32(const int8_t* input, const float* weight, float* output,
-                             size_t batch_size, size_t dims, float eps, float input_scale);
 
 void cactus_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len,
                       size_t num_heads, size_t head_dim, size_t start_pos, float theta);
 
-void cactus_rope_f32(const float* input, float* output, size_t batch_size, size_t seq_len,
-                      size_t num_heads, size_t head_dim, size_t start_pos, float theta);
-
-void cactus_rope_i8_f32_i8(const int8_t* input, int8_t* output, size_t batch_size, size_t seq_len,
-                           size_t num_heads, size_t head_dim, size_t start_pos, float theta,
-                           float input_scale, float output_scale);
-
-void cactus_softmax_f16(const __fp16* input, __fp16* output, size_t batch_size, 
+void cactus_softmax_f16(const __fp16* input, __fp16* output, size_t batch_size,
                          size_t seq_len, size_t vocab_size);
 
-void cactus_softmax_f32(const float* input, float* output, size_t batch_size, 
-                         size_t seq_len, size_t vocab_size);
-
-void cactus_silu_f32(const float* input, float* output, size_t num_elements);
 void cactus_silu_f16(const __fp16* input, __fp16* output, size_t num_elements);
-void cactus_silu_int8(const int8_t* input, int8_t* output, size_t num_elements, 
-                      float input_scale, float output_scale);
 
-void cactus_gelu_f32(const float* input, float* output, size_t num_elements);
 void cactus_gelu_f16(const __fp16* input, __fp16* output, size_t num_elements);
-void cactus_gelu_int8(const int8_t* input, int8_t* output, size_t num_elements,
-                      float input_scale, float output_scale);
 
-void cactus_gelu_f32_erf(const float* input, float* output, size_t num_elements);
 void cactus_gelu_f16_erf(const __fp16* input, __fp16* output, size_t num_elements);
-void cactus_gelu_int8_erf(
-    const int8_t* input,
-    int8_t* output,
-    size_t num_elements,
-    float scale_in,
-    float scale_out);
-
-                      
-void cactus_attention_int8(const int8_t* queries, const int8_t* keys, const int8_t* values, int8_t* output,
-                            size_t batch_size, size_t seq_len, size_t kv_seq_len, size_t num_q_heads, size_t num_kv_heads,
-                            size_t head_dim, float scale, const int8_t* mask,
-                            float q_scale, float k_scale, float v_scale, float output_scale, size_t position_offset = 0, size_t window_size = 0,
-                            bool is_causal = true);
 
 void cactus_attention_f16(const __fp16* queries, const __fp16* keys, const __fp16* values, __fp16* output,
                           size_t batch_size, size_t seq_len, size_t kv_seq_len, size_t num_q_heads, size_t num_kv_heads,
                           size_t head_dim, float scale, const __fp16* mask, size_t position_offset = 0, size_t window_size = 0,
                           bool is_causal = true);
 
-void cactus_attention_f32(const float* queries, const float* keys, const float* values, float* output,
-                          size_t batch_size, size_t seq_len, size_t kv_seq_len, size_t num_q_heads, size_t num_kv_heads,
-                          size_t head_dim, float scale, const float* mask, size_t position_offset = 0, size_t window_size = 0,
-                          bool is_causal = true);
-
-
-void cactus_conv1d_causal_depthwise_f32(
-    const float* input,
-    const float* weight,
-    float* output,
-    size_t N,
-    size_t L,
-    size_t C,
-    size_t K,
-    size_t dilation);
+void cactus_attention_hybrid_int8_fp16(
+    const __fp16* queries,        
+    const int8_t* keys_cached, 
+    const int8_t* values_cached, 
+    const float* k_scales,
+    const float* v_scales, 
+    const __fp16* keys_new, 
+    const __fp16* values_new, 
+    __fp16* output,
+    size_t batch_size, size_t seq_len, size_t cache_len, size_t new_len,
+    size_t num_q_heads, size_t num_kv_heads, size_t head_dim,
+    float scale, size_t position_offset = 0, bool is_causal = true,
+    size_t group_size = KV_QUANT_GROUP_SIZE);
 
 void cactus_conv1d_causal_depthwise_f16(
     const __fp16* input,
@@ -226,30 +113,6 @@ void cactus_conv1d_causal_depthwise_f16(
     size_t K,
     size_t dilation);
 
-void cactus_conv1d_causal_depthwise_int8(
-    const int8_t* input,
-    const int8_t* weight,
-    int8_t* output,
-    size_t N,
-    size_t L,
-    size_t C,
-    size_t K,
-    size_t dilation,
-    float input_scale,
-    float weight_scale,
-    float output_scale);
-
-void cactus_conv1d_f32_k3(
-    const float* input,
-    const float* weight,
-    float* output,
-    size_t N,
-    size_t L,
-    size_t C_in,
-    size_t C_out,
-    size_t stride
-);
-
 void cactus_conv1d_f16_k3(
     const __fp16* input,
     const __fp16* weight,
@@ -261,26 +124,8 @@ void cactus_conv1d_f16_k3(
     size_t stride
 );
 
-void cactus_conv1d_f32_k3(
-    const float* input,
-    const float* weight,
-    float* output,
-    size_t N, size_t L,
-    size_t C_in, size_t C_out,
-    size_t stride
-);
-
-void cactus_conv1d_f16_k3(
-    const __fp16* input,
-    const __fp16* weight,
-    __fp16* output,
-    size_t N, size_t L,
-    size_t C_in, size_t C_out,
-    size_t stride
-);
-
-void cactus_bilinear_interpolation_fp32(const float* input, float* output, size_t src_height, size_t src_width, size_t embed_dim,
-                                        size_t dst_height, size_t dst_width);
+void cactus_bilinear_interpolation_f16(const __fp16* input, __fp16* output, size_t src_height, size_t src_width, size_t embed_dim,
+                                       size_t dst_height, size_t dst_width);
 
 void cactus_sample_f32(const float* logits, uint32_t* output, size_t vocab_size,
                        float temperature, float top_p, size_t top_k, size_t random_seed,
@@ -291,25 +136,30 @@ void cactus_sample_f16(const __fp16* logits, uint32_t* output, size_t vocab_size
                        const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr,
                        size_t bias_count = 0);
 
-
-void cactus_concat_f32(const float* input1, const float* input2, float* output,
-                       const size_t* shape1, const size_t* shape2, const size_t* output_shape,
-                       size_t ndims, int axis);
 void cactus_concat_f16(const __fp16* input1, const __fp16* input2, __fp16* output,
                        const size_t* shape1, const size_t* shape2, const size_t* output_shape,
                        size_t ndims, int axis);
-void cactus_concat_int8(const int8_t* input1, const int8_t* input2, int8_t* output,
-                        const size_t* shape1, const size_t* shape2, const size_t* output_shape,
-                        size_t ndims, int axis);
 
 void cactus_int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f);
 void cactus_fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f);
-void cactus_dynamic_quantize_fp32_to_int8(const float* src, int8_t* dst, size_t count, float* computed_scale);
 void cactus_fp16_to_fp32(const __fp16* src, float* dst, size_t count);
 void cactus_fp32_to_fp16(const float* src, __fp16* dst, size_t count);
 void cactus_int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f);
 void cactus_fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f);
 float cactus_fp16_max_abs(const __fp16* src, size_t count);
-void cactus_int32_to_fp16_scaled(const int32_t* src, __fp16* dst, size_t count, float scale);
 
-#endif 
+void cactus_quantize_kv_fp16_to_int8(
+    const __fp16* src,
+    int8_t* dst,
+    float* scales,
+    size_t seq_len, size_t kv_heads, size_t head_dim,
+    size_t group_size = KV_QUANT_GROUP_SIZE);
+
+inline size_t kv_scales_count(size_t seq_len, size_t kv_heads, size_t head_dim, size_t group_size = KV_QUANT_GROUP_SIZE) {
+    size_t num_groups = (head_dim + group_size - 1) / group_size;
+    return seq_len * kv_heads * num_groups;
+}
+
+void cactus_unpack_int4_to_int8(const uint8_t* packed, int8_t* unpacked, size_t unpacked_count);
+
+#endif