cui-llama.rn 1.3.5 → 1.3.6

package/cpp/llama-model.h

@@ -0,0 +1,391 @@
+#pragma once
+
+#include "llama.h"
+#include "llama-arch.h"
+#include "llama-hparams.h"
+#include "llama-vocab.h"
+#include "llama-mmap.h"
+
+#include "ggml-cpp.h"
+
+#include <vector>
+
+// available models
+// TODO: this enum does not follow the enum naming convention
+enum llm_type {
+    MODEL_UNKNOWN,
+    MODEL_14M,
+    MODEL_17M,
+    MODEL_22M,
+    MODEL_33M,
+    MODEL_60M,
+    MODEL_70M,
+    MODEL_80M,
+    MODEL_109M,
+    MODEL_137M,
+    MODEL_160M,
+    MODEL_220M,
+    MODEL_250M,
+    MODEL_270M,
+    MODEL_335M,
+    MODEL_410M,
+    MODEL_450M,
+    MODEL_770M,
+    MODEL_780M,
+    MODEL_0_5B,
+    MODEL_1B,
+    MODEL_1_3B,
+    MODEL_1_4B,
+    MODEL_1_5B,
+    MODEL_1_6B,
+    MODEL_2B,
+    MODEL_2_8B,
+    MODEL_3B,
+    MODEL_4B,
+    MODEL_6B,
+    MODEL_6_9B,
+    MODEL_7B,
+    MODEL_8B,
+    MODEL_9B,
+    MODEL_11B,
+    MODEL_12B,
+    MODEL_13B,
+    MODEL_14B,
+    MODEL_15B,
+    MODEL_16B,
+    MODEL_20B,
+    MODEL_30B,
+    MODEL_32B,
+    MODEL_34B,
+    MODEL_35B,
+    MODEL_40B,
+    MODEL_65B,
+    MODEL_70B,
+    MODEL_236B,
+    MODEL_314B,
+    MODEL_671B,
+    MODEL_SMALL,
+    MODEL_MEDIUM,
+    MODEL_LARGE,
+    MODEL_XL,
+    MODEL_A1_7B,
+    MODEL_A2_7B,
+    MODEL_8x7B,
+    MODEL_8x22B,
+    MODEL_16x12B,
+    MODEL_10B_128x3_66B,
+    MODEL_57B_A14B,
+    MODEL_27B,
+};
+
+struct llama_layer_posnet {
+    // resnet
+    struct lm_ggml_tensor * norm1   = nullptr;
+    struct lm_ggml_tensor * norm1_b = nullptr;
+
+    struct lm_ggml_tensor * conv1   = nullptr;
+    struct lm_ggml_tensor * conv1_b = nullptr;
+
+    struct lm_ggml_tensor * norm2   = nullptr;
+    struct lm_ggml_tensor * norm2_b = nullptr;
+
+    struct lm_ggml_tensor * conv2   = nullptr;
+    struct lm_ggml_tensor * conv2_b = nullptr;
+
+    // attention
+    struct lm_ggml_tensor * attn_norm   = nullptr;
+    struct lm_ggml_tensor * attn_norm_b = nullptr;
+
+    struct lm_ggml_tensor * attn_q   = nullptr;
+    struct lm_ggml_tensor * attn_q_b = nullptr;
+
+    struct lm_ggml_tensor * attn_k   = nullptr;
+    struct lm_ggml_tensor * attn_k_b = nullptr;
+
+    struct lm_ggml_tensor * attn_v   = nullptr;
+    struct lm_ggml_tensor * attn_v_b = nullptr;
+
+    struct lm_ggml_tensor * attn_o   = nullptr;
+    struct lm_ggml_tensor * attn_o_b = nullptr;
+
+    // normalize
+    struct lm_ggml_tensor * norm   = nullptr;
+    struct lm_ggml_tensor * norm_b = nullptr;
+};
+
+struct llama_layer_convnext {
+    struct lm_ggml_tensor * dw   = nullptr;
+    struct lm_ggml_tensor * dw_b = nullptr;
+
+    struct lm_ggml_tensor * norm   = nullptr;
+    struct lm_ggml_tensor * norm_b = nullptr;
+
+    struct lm_ggml_tensor * pw1   = nullptr;
+    struct lm_ggml_tensor * pw1_b = nullptr;
+
+    struct lm_ggml_tensor * pw2   = nullptr;
+    struct lm_ggml_tensor * pw2_b = nullptr;
+
+    struct lm_ggml_tensor * gamma = nullptr;
+};
+
+struct llama_layer {
+    // normalization
+    struct lm_ggml_tensor * attn_norm       = nullptr;
+    struct lm_ggml_tensor * attn_norm_b     = nullptr;
+    struct lm_ggml_tensor * attn_norm_2     = nullptr;
+    struct lm_ggml_tensor * attn_norm_2_b   = nullptr;
+    struct lm_ggml_tensor * attn_q_norm     = nullptr;
+    struct lm_ggml_tensor * attn_q_norm_b   = nullptr;
+    struct lm_ggml_tensor * attn_k_norm     = nullptr;
+    struct lm_ggml_tensor * attn_k_norm_b   = nullptr;
+    struct lm_ggml_tensor * attn_out_norm   = nullptr;
+    struct lm_ggml_tensor * attn_out_norm_b = nullptr;
+    struct lm_ggml_tensor * attn_q_a_norm   = nullptr;
+    struct lm_ggml_tensor * attn_kv_a_norm  = nullptr;
+    struct lm_ggml_tensor * attn_sub_norm   = nullptr;
+    struct lm_ggml_tensor * attn_post_norm  = nullptr;
+    struct lm_ggml_tensor * ffn_sub_norm    = nullptr;
+    struct lm_ggml_tensor * attn_norm_cross = nullptr;
+    struct lm_ggml_tensor * attn_norm_enc   = nullptr;
+
+    // attention
+    struct lm_ggml_tensor * wq        = nullptr;
+    struct lm_ggml_tensor * wk        = nullptr;
+    struct lm_ggml_tensor * wv        = nullptr;
+    struct lm_ggml_tensor * wo        = nullptr;
+    struct lm_ggml_tensor * wqkv      = nullptr;
+    struct lm_ggml_tensor * wq_a      = nullptr;
+    struct lm_ggml_tensor * wq_b      = nullptr;
+    struct lm_ggml_tensor * wkv_a_mqa = nullptr;
+    struct lm_ggml_tensor * wkv_b     = nullptr;
+    struct lm_ggml_tensor * wq_cross  = nullptr;
+    struct lm_ggml_tensor * wk_cross  = nullptr;
+    struct lm_ggml_tensor * wv_cross  = nullptr;
+    struct lm_ggml_tensor * wo_cross  = nullptr;
+    struct lm_ggml_tensor * wq_enc    = nullptr;
+    struct lm_ggml_tensor * wk_enc    = nullptr;
+    struct lm_ggml_tensor * wv_enc    = nullptr;
+    struct lm_ggml_tensor * wo_enc    = nullptr;
+
+    // attention bias
+    struct lm_ggml_tensor * bq   = nullptr;
+    struct lm_ggml_tensor * bk   = nullptr;
+    struct lm_ggml_tensor * bv   = nullptr;
+    struct lm_ggml_tensor * bo   = nullptr;
+    struct lm_ggml_tensor * bqkv = nullptr;
+
+    // relative position bias
+    struct lm_ggml_tensor * attn_rel_b       = nullptr;
+    struct lm_ggml_tensor * attn_rel_b_enc   = nullptr;
+    struct lm_ggml_tensor * attn_rel_b_cross = nullptr;
+
+    // normalization
+    struct lm_ggml_tensor * ffn_norm         = nullptr;
+    struct lm_ggml_tensor * ffn_norm_b       = nullptr;
+    struct lm_ggml_tensor * ffn_post_norm    = nullptr;
+    struct lm_ggml_tensor * layer_out_norm   = nullptr;
+    struct lm_ggml_tensor * layer_out_norm_b = nullptr;
+    struct lm_ggml_tensor * ffn_norm_exps    = nullptr;
+    struct lm_ggml_tensor * ffn_norm_enc     = nullptr;
+
+    // ff
+    struct lm_ggml_tensor * ffn_gate     = nullptr; // w1
+    struct lm_ggml_tensor * ffn_down     = nullptr; // w2
+    struct lm_ggml_tensor * ffn_up       = nullptr; // w3
+    struct lm_ggml_tensor * ffn_gate_enc = nullptr;
+    struct lm_ggml_tensor * ffn_down_enc = nullptr;
+    struct lm_ggml_tensor * ffn_up_enc   = nullptr;
+
+    // ff MoE
+    struct lm_ggml_tensor * ffn_gate_inp  = nullptr;
+    struct lm_ggml_tensor * ffn_gate_exps = nullptr;
+    struct lm_ggml_tensor * ffn_down_exps = nullptr;
+    struct lm_ggml_tensor * ffn_up_exps   = nullptr;
+
+    // ff shared expert (shexp)
+    struct lm_ggml_tensor * ffn_gate_inp_shexp = nullptr;
+    struct lm_ggml_tensor * ffn_gate_shexp     = nullptr;
+    struct lm_ggml_tensor * ffn_down_shexp     = nullptr;
+    struct lm_ggml_tensor * ffn_up_shexp       = nullptr;
+
+    // ff bias
+    struct lm_ggml_tensor * ffn_gate_b = nullptr;
+    struct lm_ggml_tensor * ffn_down_b = nullptr; // b2
+    struct lm_ggml_tensor * ffn_up_b   = nullptr; // b3
+    struct lm_ggml_tensor * ffn_act    = nullptr;
+    struct lm_ggml_tensor * ffn_exp_probs_b = nullptr;
+
+    // mamba proj
+    struct lm_ggml_tensor * ssm_in  = nullptr;
+    struct lm_ggml_tensor * ssm_x   = nullptr;
+    struct lm_ggml_tensor * ssm_dt  = nullptr;
+    struct lm_ggml_tensor * ssm_out = nullptr;
+
+    // mamba
+    struct lm_ggml_tensor * ssm_conv1d = nullptr;
+    struct lm_ggml_tensor * ssm_a      = nullptr;
+    struct lm_ggml_tensor * ssm_d      = nullptr;
+
+    // mamba bias
+    struct lm_ggml_tensor * ssm_conv1d_b = nullptr;
+    struct lm_ggml_tensor * ssm_dt_b     = nullptr;
+
+    // rwkv
+    struct lm_ggml_tensor * time_mix_w1         = nullptr;
+    struct lm_ggml_tensor * time_mix_w2         = nullptr;
+    struct lm_ggml_tensor * time_mix_lerp_x     = nullptr;
+    struct lm_ggml_tensor * time_mix_lerp_w     = nullptr;
+    struct lm_ggml_tensor * time_mix_lerp_k     = nullptr;
+    struct lm_ggml_tensor * time_mix_lerp_v     = nullptr;
+    struct lm_ggml_tensor * time_mix_lerp_r     = nullptr;
+    struct lm_ggml_tensor * time_mix_lerp_g     = nullptr;
+
+    struct lm_ggml_tensor * time_mix_first      = nullptr;
+    struct lm_ggml_tensor * time_mix_decay      = nullptr;
+    struct lm_ggml_tensor * time_mix_decay_w1   = nullptr;
+    struct lm_ggml_tensor * time_mix_decay_w2   = nullptr;
+    struct lm_ggml_tensor * time_mix_key        = nullptr;
+    struct lm_ggml_tensor * time_mix_value      = nullptr;
+    struct lm_ggml_tensor * time_mix_receptance = nullptr;
+    struct lm_ggml_tensor * time_mix_gate       = nullptr;
+
+    struct lm_ggml_tensor * time_mix_ln     = nullptr;
+    struct lm_ggml_tensor * time_mix_ln_b   = nullptr;
+    struct lm_ggml_tensor * time_mix_output = nullptr;
+
+    struct lm_ggml_tensor * channel_mix_lerp_k = nullptr;
+    struct lm_ggml_tensor * channel_mix_lerp_r = nullptr;
+
+    struct lm_ggml_tensor * channel_mix_key        = nullptr;
+    struct lm_ggml_tensor * channel_mix_receptance = nullptr;
+    struct lm_ggml_tensor * channel_mix_value      = nullptr;
+
+    // long rope factors
+    struct lm_ggml_tensor * rope_long  = nullptr;
+    struct lm_ggml_tensor * rope_short = nullptr;
+    struct lm_ggml_tensor * rope_freqs = nullptr;
+
+    // bitnet scale
+    struct lm_ggml_tensor * wq_scale       = nullptr;
+    struct lm_ggml_tensor * wk_scale       = nullptr;
+    struct lm_ggml_tensor * wv_scale       = nullptr;
+    struct lm_ggml_tensor * wo_scale       = nullptr;
+    struct lm_ggml_tensor * ffn_gate_scale = nullptr;
+    struct lm_ggml_tensor * ffn_up_scale   = nullptr;
+    struct lm_ggml_tensor * ffn_down_scale = nullptr;
+
+    struct llama_layer_posnet posnet;
+
+    struct llama_layer_convnext convnext;
+};
+
+struct llama_model {
+    llm_type type = MODEL_UNKNOWN;
+    llm_arch arch = LLM_ARCH_UNKNOWN;
+
+    llama_ftype ftype = LLAMA_FTYPE_ALL_F32;
+
+    std::string name = "n/a";
+
+    llama_hparams hparams = {};
+    llama_vocab   vocab;
+
+    struct lm_ggml_tensor * tok_embd   = nullptr;
+    struct lm_ggml_tensor * type_embd  = nullptr;
+    struct lm_ggml_tensor * pos_embd   = nullptr;
+    struct lm_ggml_tensor * tok_norm   = nullptr;
+    struct lm_ggml_tensor * tok_norm_b = nullptr;
+
+    struct lm_ggml_tensor * output_norm     = nullptr;
+    struct lm_ggml_tensor * output_norm_b   = nullptr;
+    struct lm_ggml_tensor * output          = nullptr;
+    struct lm_ggml_tensor * output_b        = nullptr;
+    struct lm_ggml_tensor * output_norm_enc = nullptr;
+
+    // classifier
+    struct lm_ggml_tensor * cls       = nullptr;
+    struct lm_ggml_tensor * cls_b     = nullptr;
+    struct lm_ggml_tensor * cls_out   = nullptr;
+    struct lm_ggml_tensor * cls_out_b = nullptr;
+
+    struct lm_ggml_tensor * conv1d   = nullptr;
+    struct lm_ggml_tensor * conv1d_b = nullptr;
+
+    std::vector<llama_layer> layers;
+
+    // gguf metadata
+    std::unordered_map<std::string, std::string> lm_gguf_kv;
+
+    llama_split_mode split_mode;
+    int main_gpu;
+    int n_gpu_layers;
+
+    std::vector<std::string> rpc_servers;
+
+    // list of devices used in this model
+    std::vector<lm_ggml_backend_dev_t> devices;
+
+
+    // lists of buffer types used for each layer
+    using buft_list_t = std::vector<std::pair<lm_ggml_backend_dev_t, lm_ggml_backend_buffer_type_t>>;
+    buft_list_t cpu_buft_list;
+    std::map<lm_ggml_backend_dev_t, buft_list_t> gpu_buft_list;
+
+    struct layer_dev {
+        lm_ggml_backend_dev_t dev;
+        buft_list_t * buft_list;
+    };
+
+    layer_dev dev_input = {};
+    layer_dev dev_output = {};
+    std::vector<layer_dev> dev_layer;
+
+    // contexts where the model tensors metadata is stored
+    std::vector<lm_ggml_context_ptr> ctxs;
+
+    // the model memory buffers for the tensor data
+    std::vector<lm_ggml_backend_buffer_ptr> bufs;
+
+    // model memory mapped files
+    llama_mmaps mappings;
+
+    // objects representing data potentially being locked in memory
+    llama_mlocks mlock_bufs;
+    llama_mlocks mlock_mmaps;
+
+    // for quantize-stats only
+    std::vector<std::pair<std::string, struct lm_ggml_tensor *>> tensors_by_name;
+
+    int64_t t_load_us  = 0;
+    int64_t t_start_us = 0;
+
+    // total number of parameters in the model
+    uint64_t n_elements = 0;
+
+    // total size of all the tensors in the model in bytes
+    size_t  n_bytes     = 0;
+};
+
+const char * llm_type_name(llm_type type);
+
+std::string llama_model_arch_name (const llama_model & model);
+std::string llama_model_type_name (const llama_model & model);
+std::string llama_model_ftype_name(const llama_model & model);
+
+// used by llama_adapter_cvec
+lm_ggml_backend_buffer_type_t llama_model_select_buft(const llama_model & model, int il);
+
+// used by llama_adapter_lora
+struct lm_ggml_tensor * llama_model_get_tensor(const struct llama_model & model, const char * name);
+
+size_t llama_model_max_nodes(const llama_model & model);
+
+struct llama_model_loader;
+
+// TODO: become llama_model methods
+void llm_load_stats     (llama_model_loader & ml, llama_model & model);
+void llm_load_arch      (llama_model_loader & ml, llama_model & model);
+void llm_load_hparams   (llama_model_loader & ml, llama_model & model);
+void llm_load_vocab     (llama_model_loader & ml, llama_model & model);
+void llm_load_print_meta(llama_model_loader & ml, llama_model & model);

package/cpp/llama-sampling.cpp

@@ -1,5 +1,6 @@
 #include "llama-sampling.h"
 
+#include "llama-impl.h"
 #include "llama-vocab.h"
 #include "llama-grammar.h"
 
@@ -14,6 +15,118 @@
 #include <numeric>
 #include <random>
 #include <unordered_map>
+#include <stdexcept>
+
+// the ring buffer works similarly to std::deque, but with a fixed capacity
+template<typename T>
+struct ring_buffer {
+    ring_buffer(size_t cap) : capacity(cap), data(cap) {}
+
+    T & front() {
+        if (sz == 0) {
+            throw std::runtime_error("ring buffer is empty");
+        }
+        return data[first];
+    }
+
+    const T & front() const {
+        if (sz == 0) {
+            throw std::runtime_error("ring buffer is empty");
+        }
+        return data[first];
+    }
+
+    T & back() {
+        if (sz == 0) {
+            throw std::runtime_error("ring buffer is empty");
+        }
+        return data[pos];
+    }
+
+    const T & back() const {
+        if (sz == 0) {
+            throw std::runtime_error("ring buffer is empty");
+        }
+        return data[pos];
+    }
+
+    void push_back(const T & value) {
+        if (capacity == 0) {
+            throw std::runtime_error("ring buffer: capacity is zero");
+        }
+
+        if (sz == capacity) {
+            // advance the start when buffer is full
+            first = (first + 1) % capacity;
+        } else {
+            sz++;
+        }
+        data[pos] = value;
+        pos = (pos + 1) % capacity;
+    }
+
+    T pop_front() {
+        if (sz == 0) {
+            throw std::runtime_error("ring buffer is empty");
+        }
+        T value = data[first];
+        first = (first + 1) % capacity;
+        sz--;
+        return value;
+    }
+
+    //T & operator[](size_t i) {
+    //    if (i >= sz) {
+    //        throw std::runtime_error("ring buffer: index out of bounds");
+    //    }
+    //    return data[(first + i) % capacity];
+    //}
+
+    //const T & at(size_t i) const {
+    //    if (i >= sz) {
+    //        throw std::runtime_error("ring buffer: index out of bounds");
+    //    }
+    //    return data[(first + i) % capacity];
+    //}
+
+    const T & rat(size_t i) const {
+        if (i >= sz) {
+            throw std::runtime_error("ring buffer: index out of bounds");
+        }
+        return data[(first + sz - i - 1) % capacity];
+    }
+
+    std::vector<T> to_vector() const {
+        std::vector<T> result;
+        result.reserve(sz);
+        for (size_t i = 0; i < sz; i++) {
+            result.push_back(data[(first + i) % capacity]);
+        }
+        return result;
+    }
+
+    void clear() {
+        // here only reset the status of the buffer
+        sz = 0;
+        first = 0;
+        pos = 0;
+    }
+
+    bool empty() const {
+        return sz == 0;
+    }
+
+    size_t size() const {
+        return sz;
+    }
+
+    size_t capacity = 0;
+    size_t sz = 0;
+    size_t first = 0;
+    size_t pos = 0;
+
+    std::vector<T> data;
+};
 
 static int llama_sample_dist(llama_token_data_array * cur_p, std::mt19937 & rng) {
     // iterator for the probabilities
@@ -144,7 +257,7 @@ static void llama_sampler_top_k_impl(llama_token_data_array * cur_p, int32_t k)
             for (int i = 0; i < (int)cur_p->size; ++i) {
                 const float val = cur_p->data[i].logit;
                 int ib = int(bucket_scale * val + bucket_inter); //nbuckets * (val - bucket_low) / (bucket_high - bucket_low);
-                ib = std::max(0, std::min(nbuckets-1, ib));
+                ib = std::max(0, std::min(nbuckets - 1, ib));
                 bucket_idx[i] = ib;
                 ++histo[ib];
             }
@@ -167,13 +280,13 @@ static void llama_sampler_top_k_impl(llama_token_data_array * cur_p, int32_t k)
             for (int i = 0; i < (int)cur_p->size; ++i) {
                 int j = bucket_idx[i];
                 if (j >= ib) {
-                    *bucket_ptrs[nbuckets-1-j]++ = cur_p->data[i];
+                    *bucket_ptrs[nbuckets - 1 - j]++ = cur_p->data[i];
                 }
             }
 
             ptr = tmp_tokens.data();
             int ndone = 0;
-            for (int j = nbuckets-1; j > ib; --j) {
+            for (int j = nbuckets - 1; j > ib; --j) {
                 std::sort(ptr, ptr + histo[j], comp);
                 ptr += histo[j];
                 ndone += histo[j];
@@ -1720,7 +1833,7 @@ static void llama_sampler_dry_apply(struct llama_sampler * smpl, llama_token_dat
                 ctx->dry_repeat_count[last - k] = std::min(n, rep_limit);
                 if (n > 0) {
                     lt = k;
-                    rt = k+n-1;
+                    rt = k + n - 1;
                 }
             } else {
                 // If k is inside the current Z-box, consider two cases.

package/cpp/llama-vocab.cpp

@@ -1,5 +1,7 @@
 #include "llama-vocab.h"
 
+#include "llama-impl.h"
+
 #include "unicode.h"
 
 #include <algorithm>
@@ -16,22 +18,6 @@
 // helpers
 //
 
-LLAMA_ATTRIBUTE_FORMAT(1, 2)
-static std::string format(const char * fmt, ...) {
-    va_list ap;
-    va_list ap2;
-    va_start(ap, fmt);
-    va_copy(ap2, ap);
-    int size = vsnprintf(NULL, 0, fmt, ap);
-    LM_GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT
-    std::vector<char> buf(size + 1);
-    int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);
-    LM_GGML_ASSERT(size2 == size);
-    va_end(ap2);
-    va_end(ap);
-    return std::string(buf.data(), size);
-}
-
 struct naive_trie {
     naive_trie() : has_value(false), value(0) {
     }
@@ -396,6 +382,13 @@ struct llm_tokenizer_bpe : llm_tokenizer {
                     "\\p{N}+",
                 };
                 break;
+            case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM:
+                regex_exprs = {
+                    "\\p{N}{1,3}",
+                    "[一-龥぀-ゟ゠-ヿ]+",
+                    "[!\"#$%&'()*+,\\-./:;<=>?@\\[\\\\\\]^_`{|}~][A-Za-z]+|[^\r\n\\p{L}\\p{P}\\p{S}]?[\\p{L}\\p{M}]+| ?[\\p{P}\\p{S}]+[\r\n]*|\\s*[\r\n]+|\\s+(?!\\S)|\\s+",
+                };
+                break;
             case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER:
                 regex_exprs = {
                     "[\r\n]",
@@ -504,7 +497,7 @@ struct llm_tokenizer_bpe_session {
 
     bool append_bos(std::vector<llama_vocab::id> & output) const {
         if (vocab.tokenizer_add_bos) {
-            LM_GGML_ASSERT(vocab.special_bos_id != -1);
+            LM_GGML_ASSERT(vocab.special_bos_id != LLAMA_TOKEN_NULL);
             output.push_back(vocab.special_bos_id);
             return true;
         }
@@ -513,7 +506,7 @@ struct llm_tokenizer_bpe_session {
 
     bool append_eos(std::vector<llama_vocab::id> & output) const {
         if (vocab.tokenizer_add_eos) {
-            LM_GGML_ASSERT(vocab.special_eos_id != -1);
+            LM_GGML_ASSERT(vocab.special_eos_id != LLAMA_TOKEN_NULL);
             output.push_back(vocab.special_eos_id);
             return true;
         }
@@ -1410,7 +1403,7 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<
                         if (source == 0) {
                             buffer.erase_after(buffer.before_begin());
                         } else {
-                            buffer.erase_after(std::next(buffer.begin(), (source-1)));
+                            buffer.erase_after(std::next(buffer.begin(), (source - 1)));
                         }
 
                         // repeat for the right side
@@ -1424,7 +1417,7 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<
                         if (source == 0) {
                             buffer.erase_after(buffer.before_begin());
                         } else {
-                            buffer.erase_after(std::next(buffer.begin(), (source-1)));
+                            buffer.erase_after(std::next(buffer.begin(), (source - 1)));
                         }
                         break;
                     }
@@ -1461,7 +1454,7 @@ std::vector<llama_vocab::id> llama_tokenize_internal(
                 bool is_prev_special = true;  // prefix with space if first token
 
                 if (add_special && vocab.tokenizer_add_bos) {
-                    LM_GGML_ASSERT(vocab.special_bos_id != -1);
+                    LM_GGML_ASSERT(vocab.special_bos_id != LLAMA_TOKEN_NULL);
                     output.push_back(vocab.special_bos_id);
                     is_prev_special = true;
                 }
@@ -1496,7 +1489,7 @@ std::vector<llama_vocab::id> llama_tokenize_internal(
                 }
 
                 if (add_special && vocab.tokenizer_add_eos) {
-                    LM_GGML_ASSERT(vocab.special_eos_id != -1);
+                    LM_GGML_ASSERT(vocab.special_eos_id != LLAMA_TOKEN_NULL);
                     output.push_back(vocab.special_eos_id);
                 }
             } break;
@@ -1529,7 +1522,7 @@ std::vector<llama_vocab::id> llama_tokenize_internal(
         case LLAMA_VOCAB_TYPE_WPM:
             {
                 if (add_special) {
-                    LM_GGML_ASSERT(vocab.special_cls_id != -1);
+                    LM_GGML_ASSERT(vocab.special_cls_id != LLAMA_TOKEN_NULL);
                     output.push_back(vocab.special_cls_id);
                 }
 
@@ -1549,14 +1542,14 @@ std::vector<llama_vocab::id> llama_tokenize_internal(
                 }
 
                 if (add_special) {
-                    LM_GGML_ASSERT(vocab.special_sep_id != -1);
+                    LM_GGML_ASSERT(vocab.special_sep_id != LLAMA_TOKEN_NULL);
                     output.push_back(vocab.special_sep_id);
                 }
             } break;
         case LLAMA_VOCAB_TYPE_UGM:
             {
                 if (add_special && vocab.tokenizer_add_bos) {
-                    LM_GGML_ASSERT(vocab.special_bos_id != -1);
+                    LM_GGML_ASSERT(vocab.special_bos_id != LLAMA_TOKEN_NULL);
                     output.push_back(vocab.special_bos_id);
                 }
                 llm_tokenizer_ugm_session session(vocab);
@@ -1581,7 +1574,7 @@ std::vector<llama_vocab::id> llama_tokenize_internal(
                 }
 
                 if (add_special && vocab.tokenizer_add_eos) {
-                    LM_GGML_ASSERT(vocab.special_eos_id != -1);
+                    LM_GGML_ASSERT(vocab.special_eos_id != LLAMA_TOKEN_NULL);
                     output.push_back(vocab.special_eos_id);
                 }
             } break;
@@ -1649,7 +1642,7 @@ llama_token_attr llama_token_get_attr_impl(const struct llama_vocab & vocab, lla
 }
 
 bool llama_token_is_eog_impl(const struct llama_vocab & vocab, llama_token token) {
-    return token != -1 && vocab.special_eog_ids.count(token) > 0;
+    return token != LLAMA_TOKEN_NULL && vocab.special_eog_ids.count(token) > 0;
 }
 
 bool llama_token_is_control_impl(const struct llama_vocab & vocab, llama_token token) {
@@ -1888,7 +1881,7 @@ int32_t llama_detokenize_impl(
     }
 
     if (remove_special && vocab.tokenizer_add_eos) {
-        if (n_tokens > 0 && tokens[n_tokens-1] == vocab.special_eos_id) {
+        if (n_tokens > 0 && tokens[n_tokens - 1] == vocab.special_eos_id) {
             n_tokens--;
         }
     }