node-llama-cpp 3.3.2 → 3.4.1

package/llama/addon/AddonContext.cpp

@@ -1,5 +1,6 @@
 #include <thread>
 #include <algorithm>
+#include <cmath>
 #include "common/common.h"
 #include "llama-grammar.h"
 #include "llama.h"
@@ -104,13 +105,13 @@ class AddonContextLoadContextWorker : public Napi::AsyncWorker {
 
         void Execute() {
             try {
-                context->ctx = llama_new_context_with_model(context->model->model, context->context_params);
+                context->ctx = llama_init_from_model(context->model->model, context->context_params);
 
                 context->contextLoaded = context->ctx != nullptr && context->ctx != NULL;
             } catch (const std::exception& e) {
                 SetError(e.what());
             } catch(...) {
-                SetError("Unknown error when calling \"llama_new_context_with_model\"");
+                SetError("Unknown error when calling \"llama_init_from_model\"");
             }
         }
         void OnOK() {
@@ -190,6 +191,14 @@ class AddonContextSampleTokenWorker : public Napi::AsyncWorker {
     public:
         AddonContext* ctx;
         AddonSampler* sampler;
+        bool arrayResult = false;
+        bool returnProbabilities = false;
+        bool returnConfidence = false;
+        float tokenConfidence = -1;
+        bool has_probabilities = false;
+        size_t probabilities_size;
+        llama_token * probabilities_tokens;
+        float * probabilities_probs;
         int32_t batchLogitIndex;
         llama_token result;
         bool no_output = false;
@@ -202,11 +211,19 @@ class AddonContextSampleTokenWorker : public Napi::AsyncWorker {
 
             batchLogitIndex = info[0].As<Napi::Number>().Int32Value();
             sampler = Napi::ObjectWrap<AddonSampler>::Unwrap(info[1].As<Napi::Object>());
+            arrayResult = info.Length() > 2 && info[2].IsBoolean();
+            returnProbabilities = arrayResult ? info[2].As<Napi::Boolean>().Value() : false;
+            returnConfidence = arrayResult && info.Length() > 3 && info[3].IsBoolean() ? info[3].As<Napi::Boolean>().Value() : false;
             sampler->Ref();
         }
         ~AddonContextSampleTokenWorker() {
             ctx->Unref();
             sampler->Unref();
+
+            if (has_probabilities) {
+                delete[] probabilities_tokens;
+                delete[] probabilities_probs;
+            }
         }
 
         Napi::Promise GetPromise() {
@@ -235,11 +252,11 @@ class AddonContextSampleTokenWorker : public Napi::AsyncWorker {
             sampler->rebuildChainIfNeeded();
 
             const auto * logits = llama_get_logits_ith(ctx->ctx, batchLogitIndex);
-            const int n_vocab = llama_n_vocab(ctx->model->model);
+            const int n_vocab = llama_vocab_n_tokens(ctx->model->vocab);
 
             auto & candidates = sampler->tokenCandidates;
             for (llama_token token_id = 0; token_id < n_vocab; token_id++) {
-                candidates[token_id] = llama_token_data{token_id, logits[token_id], 0.0f};;
+                candidates[token_id] = llama_token_data{token_id, logits[token_id], 0.0f};
             }
 
             llama_token_data_array cur_p = {
@@ -257,18 +274,111 @@ class AddonContextSampleTokenWorker : public Napi::AsyncWorker {
             }
 
             auto new_token_id = cur_p.data[cur_p.selected].id;
+
+            if (returnProbabilities || returnConfidence) {
+                if (!cur_p.sorted) {
+                    std::sort(cur_p.data, cur_p.data + cur_p.size, [](const llama_token_data & a, const llama_token_data & b) {
+                        return a.logit > b.logit;
+                    });
+                    cur_p.sorted = true;
+
+                    for (size_t i = 0; i < cur_p.size; i++) {
+                        if (cur_p.data[i].id == new_token_id) {
+                            cur_p.selected = i;
+                            break;
+                        }
+                    }
+                }
+            }
+
+            if (returnProbabilities) {
+                probabilities_size = cur_p.size;
+                probabilities_tokens = new llama_token[probabilities_size];
+                probabilities_probs = new float[probabilities_size];
+                float maxLogit = cur_p.size > 0 ? cur_p.data[0].logit : -INFINITY;
+
+                for (size_t i = 0; i < cur_p.size; i++) {
+                    auto logit = cur_p.data[i].logit;
+
+                    probabilities_tokens[i] = cur_p.data[i].id;
+                    probabilities_probs[i] = logit;
+
+                    if (logit > maxLogit) {
+                        maxLogit = logit;
+                    }
+                }
+
+                if (probabilities_size > 0 && maxLogit != -INFINITY) {
+                    float sum = 0.0f;
+                    for (size_t i = 0; i < probabilities_size; i++) {
+                        float prob = expf(probabilities_probs[i] - maxLogit);
+                        probabilities_probs[i] = prob;
+                        sum += prob;
+                    }
+
+                    for (size_t i = 0; i < probabilities_size; i++) {
+                        probabilities_probs[i] /= sum;
+                    }
+                }
+
+                has_probabilities = true;
+            }
+
+            if (returnConfidence) {
+                if (has_probabilities && cur_p.selected < probabilities_size) {
+                    tokenConfidence = probabilities_probs[cur_p.selected];
+                } else {
+                    float maxLogit = cur_p.data[0].logit;
+                    float sum = 0.0f;
+                    for (size_t i = 0; i < cur_p.size; i++) {
+                        auto logit = cur_p.data[i].logit;
+
+                        if (logit > maxLogit) {
+                            maxLogit = logit;
+                        }
+                    }
+
+                    for (size_t i = 0; i < cur_p.size; i++) {
+                        sum += expf(cur_p.data[i].logit - maxLogit);
+                    }
+
+                    tokenConfidence = expf(cur_p.data[cur_p.selected].logit - maxLogit) / sum;
+                }
+            }
+
             sampler->acceptToken(new_token_id);
             result = new_token_id;
         }
         void OnOK() {
+            Napi::Number resultToken;
             if (no_output) {
-                Napi::Number resultValue = Napi::Number::New(Env(), -1);
-                deferred.Resolve(resultValue);
+                resultToken = Napi::Number::New(Env(), -1);
+            } else {
+                resultToken = Napi::Number::New(Env(), static_cast<uint32_t>(result));
+            }
+
+            if (!arrayResult) {
+                deferred.Resolve(resultToken);
                 return;
             }
 
-            Napi::Number resultValue = Napi::Number::New(Env(), static_cast<uint32_t>(result));
-            deferred.Resolve(resultValue);
+            Napi::Array resultArray = Napi::Array::New(Env(), 2);
+            resultArray.Set(Napi::Number::New(Env(), 0), resultToken);
+
+            if (has_probabilities) {
+                Napi::Array probabilities = Napi::Array::New(Env(), probabilities_size * 2);
+                for (size_t i = 0; i < probabilities_size; i++) {
+                    probabilities.Set(i * 2, Napi::Number::New(Env(), probabilities_tokens[i]));
+                    probabilities.Set(i * 2 + 1, Napi::Number::New(Env(), probabilities_probs[i]));
+                }
+                resultArray.Set(1, probabilities);
+            }
+
+            if (returnConfidence && tokenConfidence != -1) {
+                resultArray.Set(2, Napi::Number::New(Env(), tokenConfidence));
+            }
+
+            deferred.Resolve(resultArray);
         }
         void OnError(const Napi::Error& err) {
             deferred.Reject(err.Value());
@@ -305,6 +415,10 @@ AddonContext::AddonContext(const Napi::CallbackInfo& info) : Napi::ObjectWrap<Ad
             context_params.embeddings = options.Get("embeddings").As<Napi::Boolean>().Value();
         }
 
+        if (options.Has("ranking") && options.Get("ranking").As<Napi::Boolean>().Value()) {
+            context_params.pooling_type = LLAMA_POOLING_TYPE_RANK;
+        }
+
         if (options.Has("flashAttention")) {
             context_params.flash_attn = options.Get("flashAttention").As<Napi::Boolean>().Value();
         }
@@ -411,7 +525,7 @@ Napi::Value AddonContext::InitBatch(const Napi::CallbackInfo& info) {
     has_batch = true;
     batch_n_tokens = n_tokens;
 
-    uint64_t newBatchMemorySize = calculateBatchMemorySize(n_tokens, llama_n_embd(model->model), context_params.n_batch);
+    uint64_t newBatchMemorySize = calculateBatchMemorySize(n_tokens, llama_model_n_embd(model->model), context_params.n_batch);
     if (newBatchMemorySize > batchMemorySize) {
         adjustNapiExternalMemoryAdd(Env(), newBatchMemorySize - batchMemorySize);
         batchMemorySize = newBatchMemorySize;
@@ -441,24 +555,25 @@ Napi::Value AddonContext::AddToBatch(const Napi::CallbackInfo& info) {
     int32_t sequenceId = info[0].As<Napi::Number>().Int32Value();
     int32_t firstTokenContextIndex = info[1].As<Napi::Number>().Int32Value();
     Napi::Uint32Array tokens = info[2].As<Napi::Uint32Array>();
-    bool generateLogitAtTheEnd = info[3].As<Napi::Boolean>().Value();
+    Napi::Uint32Array tokenLogitIndexes = info[3].As<Napi::Uint32Array>();
 
     auto tokensLength = tokens.ElementLength();
+    auto tokenLogitIndexesLength = tokenLogitIndexes.ElementLength();
     GGML_ASSERT(batch.n_tokens + tokensLength <= batch_n_tokens);
 
-    for (size_t i = 0; i < tokensLength; i++) {
-        common_batch_add(batch, static_cast<llama_token>(tokens[i]), firstTokenContextIndex + i, { sequenceId }, false);
-    }
-
-    if (generateLogitAtTheEnd) {
-        batch.logits[batch.n_tokens - 1] = true;
+    Napi::Uint32Array resLogitIndexes = Napi::Uint32Array::New(info.Env(), tokenLogitIndexesLength);
 
-        auto logit_index = batch.n_tokens - 1;
-
-        return Napi::Number::From(info.Env(), logit_index);
+    for (size_t i = 0, l = 0; i < tokensLength; i++) {
+        if (l < tokenLogitIndexesLength && l < tokenLogitIndexesLength && tokenLogitIndexes[l] == i) {
+            common_batch_add(batch, static_cast<llama_token>(tokens[i]), firstTokenContextIndex + i, { sequenceId }, true);
+            resLogitIndexes[l] = batch.n_tokens - 1;
+            l++;
+        } else {
+            common_batch_add(batch, static_cast<llama_token>(tokens[i]), firstTokenContextIndex + i, { sequenceId }, false);
+        }
     }
 
-    return info.Env().Undefined();
+    return resLogitIndexes;
 }
 Napi::Value AddonContext::DisposeSequence(const Napi::CallbackInfo& info) {
     if (disposed) {
@@ -530,7 +645,7 @@ Napi::Value AddonContext::GetEmbedding(const Napi::CallbackInfo& info) {
         return info.Env().Undefined();
     }
 
-    const int n_embd = llama_n_embd(model->model);
+    const int n_embd = llama_model_n_embd(model->model);
     const enum llama_pooling_type pooling_type = llama_pooling_type(ctx);
     const auto* embeddings = pooling_type == LLAMA_POOLING_TYPE_NONE ? NULL : llama_get_embeddings_seq(ctx, 0);
     if (embeddings == NULL) {
@@ -592,11 +707,69 @@ Napi::Value AddonContext::PrintTimings(const Napi::CallbackInfo& info) {
     return info.Env().Undefined();
 }
 
+Napi::Value AddonContext::EnsureDraftContextIsCompatibleForSpeculative(const Napi::CallbackInfo& info) {
+    constexpr auto vocabSizeMaxDifference = 128; // SPEC_VOCAB_MAX_SIZE_DIFFERENCE
+    constexpr auto vocabCheckStartTokenId = 5; // SPEC_VOCAB_CHECK_START_TOKEN_ID
+
+    const AddonContext * draftContext = Napi::ObjectWrap<AddonContext>::Unwrap(info[0].As<Napi::Object>());
+    const auto currentCtx = ctx;
+    const auto draftCtx = draftContext->ctx;
+    const auto currentModel = model->model;
+    const auto draftModel = draftContext->model->model;
+    const auto currentVocab = model->vocab;
+    const auto draftVocab = draftContext->model->vocab;
+
+    if (llama_vocab_type(currentVocab) != llama_vocab_type(draftVocab)) {
+        Napi::Error::New(info.Env(), "Speculative draft model vocabulary type must match the target model vocabulary type").ThrowAsJavaScriptException();
+        return info.Env().Undefined();
+    }
+
+    if (llama_vocab_get_add_bos(currentVocab) != llama_vocab_get_add_bos(draftVocab) ||
+        llama_vocab_get_add_eos(currentVocab) != llama_vocab_get_add_eos(draftVocab) ||
+        llama_vocab_bos(currentVocab) != llama_vocab_bos(draftVocab) ||
+        llama_vocab_eos(currentVocab) != llama_vocab_eos(draftVocab)
+    ) {
+        Napi::Error::New(info.Env(), "Speculative draft model special tokens must match the target model special tokens").ThrowAsJavaScriptException();
+        return info.Env().Undefined();
+    }
+
+    const int currentModelVocabSize = llama_vocab_n_tokens(currentVocab);
+    const int draftModelVocabSize = llama_vocab_n_tokens(draftVocab);
+
+    const int vocabDiff = std::abs(currentModelVocabSize - draftModelVocabSize);
+
+    if (vocabDiff > vocabSizeMaxDifference) {
+        Napi::Error::New(
+            info.Env(),
+            std::string("Speculative draft model vocabulary must closely match the target model vocabulary size (vocabulary size difference: ") +
+            std::to_string(vocabDiff) + std::string(", max allowed: ") + std::to_string(vocabSizeMaxDifference) + std::string(")")
+        ).ThrowAsJavaScriptException();
+        return info.Env().Undefined();
+    }
+
+    const int minVocabSize = std::min(currentModelVocabSize, draftModelVocabSize);
+    for (int i = vocabCheckStartTokenId; i < minVocabSize; ++i) {
+        const char * currentTokenText = llama_vocab_get_text(currentVocab, i);
+        const char * draftTokenText = llama_vocab_get_text(draftVocab, i);
+        if (std::strcmp(currentTokenText, draftTokenText) != 0) {
+            Napi::Error::New(
+                info.Env(),
+                std::string("Speculative draft model vocabulary must match the target model vocabulary, but token ") +
+                std::to_string(i) + std::string(" content differs. Target: \"") + std::string(currentTokenText) +
+                std::string("\", Draft: \"") + std::string(draftTokenText) + std::string("")
+            ).ThrowAsJavaScriptException();
+            return info.Env().Undefined();
+        }
+    }
+
+    return info.Env().Undefined();
+}
+
 Napi::Value AddonContext::SetLora(const Napi::CallbackInfo& info) {
     AddonModelLora* lora = Napi::ObjectWrap<AddonModelLora>::Unwrap(info[0].As<Napi::Object>());
     float scale = info[1].As<Napi::Number>().FloatValue();
 
-    llama_lora_adapter_set(ctx, lora->lora_adapter, scale);
+    llama_set_adapter_lora(ctx, lora->lora_adapter, scale);
 
     return info.Env().Undefined();
 }
@@ -622,6 +795,7 @@ void AddonContext::init(Napi::Object exports) {
                 InstanceMethod("getThreads", &AddonContext::GetThreads),
                 InstanceMethod("setThreads", &AddonContext::SetThreads),
                 InstanceMethod("printTimings", &AddonContext::PrintTimings),
+                InstanceMethod("ensureDraftContextIsCompatibleForSpeculative", &AddonContext::EnsureDraftContextIsCompatibleForSpeculative),
                 InstanceMethod("setLora", &AddonContext::SetLora),
                 InstanceMethod("dispose", &AddonContext::Dispose),
             }

package/llama/addon/AddonContext.h

@@ -45,6 +45,7 @@ class AddonContext : public Napi::ObjectWrap<AddonContext> {
         Napi::Value SetThreads(const Napi::CallbackInfo& info);
 
         Napi::Value PrintTimings(const Napi::CallbackInfo& info);
+        Napi::Value EnsureDraftContextIsCompatibleForSpeculative(const Napi::CallbackInfo& info);
 
         Napi::Value SetLora(const Napi::CallbackInfo& info);
 

package/llama/addon/AddonGrammar.cpp

@@ -46,13 +46,10 @@ Napi::Value AddonGrammar::isTextCompatible(const Napi::CallbackInfo& info) {
     }
 
     const auto cpts = unicode_cpts_from_utf8(testText);
-    const llama_grammar_rules  & rules = llama_grammar_get_rules(parsed_grammar);
     llama_grammar_stacks & stacks_cur = llama_grammar_get_stacks(parsed_grammar);
 
     for (const auto & cpt : cpts) {
-        const llama_grammar_stacks stacks_prev = llama_grammar_get_stacks(parsed_grammar);
-
-        llama_grammar_accept(rules, stacks_prev, cpt, stacks_cur);
+        llama_grammar_accept(parsed_grammar, cpt);
 
         if (stacks_cur.empty()) {
             // no stacks means that the grammar failed to match at this point

package/llama/addon/AddonGrammarEvaluationState.cpp

@@ -6,13 +6,24 @@
 #include "AddonGrammar.h"
 
 AddonGrammarEvaluationState::AddonGrammarEvaluationState(const Napi::CallbackInfo& info) : Napi::ObjectWrap<AddonGrammarEvaluationState>(info) {
-    model = Napi::ObjectWrap<AddonModel>::Unwrap(info[0].As<Napi::Object>());
-    model->Ref();
+    if (info.Length() == 1) {
+        AddonGrammarEvaluationState* existingState = Napi::ObjectWrap<AddonGrammarEvaluationState>::Unwrap(info[0].As<Napi::Object>());
+        model = existingState->model;
+        model->Ref();
 
-    grammarDef = Napi::ObjectWrap<AddonGrammar>::Unwrap(info[1].As<Napi::Object>());
-    grammarDef->Ref();
+        grammarDef = existingState->grammarDef;
+        grammarDef->Ref();
 
-    sampler = llama_sampler_init_grammar(model->model, grammarDef->grammarCode.c_str(), grammarDef->rootRuleName.c_str());
+        sampler = llama_sampler_clone(existingState->sampler);
+    } else {
+        model = Napi::ObjectWrap<AddonModel>::Unwrap(info[0].As<Napi::Object>());
+        model->Ref();
+
+        grammarDef = Napi::ObjectWrap<AddonGrammar>::Unwrap(info[1].As<Napi::Object>());
+        grammarDef->Ref();
+
+        sampler = llama_sampler_init_grammar(model->vocab, grammarDef->grammarCode.c_str(), grammarDef->rootRuleName.c_str());
+    }
 }
 AddonGrammarEvaluationState::~AddonGrammarEvaluationState() {
     llama_sampler_free(sampler);

package/llama/addon/AddonModel.cpp

@@ -8,12 +8,12 @@
 #include "AddonModelData.h"
 #include "AddonModelLora.h"
 
-static Napi::Value getNapiToken(const Napi::CallbackInfo& info, llama_model* model, llama_token token) {
+static Napi::Value getNapiToken(const Napi::CallbackInfo& info, const llama_vocab* vocab, llama_token token) {
     if (token < 0 || token == LLAMA_TOKEN_NULL) {
         return Napi::Number::From(info.Env(), -1);
     }
 
-    auto tokenAttributes = llama_token_get_attr(model, token);
+    auto tokenAttributes = llama_vocab_get_attr(vocab, token);
 
     if (tokenAttributes & LLAMA_TOKEN_ATTR_UNDEFINED || tokenAttributes & LLAMA_TOKEN_ATTR_UNKNOWN) {
         return Napi::Number::From(info.Env(), -1);
@@ -22,12 +22,12 @@ static Napi::Value getNapiToken(const Napi::CallbackInfo& info, llama_model* mod
     return Napi::Number::From(info.Env(), token);
 }
 
-static Napi::Value getNapiControlToken(const Napi::CallbackInfo& info, llama_model* model, llama_token token) {
+static Napi::Value getNapiControlToken(const Napi::CallbackInfo& info, const llama_vocab* vocab, llama_token token) {
     if (token < 0) {
         return Napi::Number::From(info.Env(), -1);
     }
 
-    auto tokenAttributes = llama_token_get_attr(model, token);
+    auto tokenAttributes = llama_vocab_get_attr(vocab, token);
 
     if (!(tokenAttributes & LLAMA_TOKEN_ATTR_CONTROL) && !(tokenAttributes & LLAMA_TOKEN_ATTR_UNDEFINED)) {
         return Napi::Number::From(info.Env(), -1);
@@ -92,13 +92,14 @@ class AddonModelLoadModelWorker : public Napi::AsyncWorker {
 
         void Execute() {
             try {
-                model->model = llama_load_model_from_file(model->modelPath.c_str(), model->model_params);
+                model->model = llama_model_load_from_file(model->modelPath.c_str(), model->model_params);
+                model->vocab = llama_model_get_vocab(model->model);
 
                 model->modelLoaded = model->model != nullptr && model->model != NULL;
             } catch (const std::exception& e) {
                 SetError(e.what());
             } catch(...) {
-                SetError("Unknown error when calling \"llama_load_model_from_file\"");
+                SetError("Unknown error when calling \"llama_model_load_from_file\"");
             }
         }
         void OnOK() {
@@ -141,14 +142,14 @@ class AddonModelUnloadModelWorker : public Napi::AsyncWorker {
 
         void Execute() {
             try {
-                llama_free_model(model->model);
+                llama_model_free(model->model);
                 model->modelLoaded = false;
 
                 model->dispose();
             } catch (const std::exception& e) {
                 SetError(e.what());
             } catch(...) {
-                SetError("Unknown error when calling \"llama_free_model\"");
+                SetError("Unknown error when calling \"llama_model_free\"");
             }
         }
         void OnOK() {
@@ -190,7 +191,7 @@ class AddonModelLoadLoraWorker : public Napi::AsyncWorker {
 
         void Execute() {
             try {
-                const auto loraAdapter = llama_lora_adapter_init(modelLora->model->model, modelLora->loraFilePath.c_str());
+                const auto loraAdapter = llama_adapter_lora_init(modelLora->model->model, modelLora->loraFilePath.c_str());
 
                 if (loraAdapter == nullptr) {
                     SetError(
@@ -213,7 +214,7 @@ class AddonModelLoadLoraWorker : public Napi::AsyncWorker {
             } catch (const std::exception& e) {
                 SetError(e.what());
             } catch(...) {
-                SetError("Unknown error when calling \"llama_lora_adapter_init\"");
+                SetError("Unknown error when calling \"llama_adapter_lora_init\"");
             }
         }
         void OnOK() {
@@ -359,7 +360,7 @@ void AddonModel::dispose() {
     disposed = true;
     if (modelLoaded) {
         modelLoaded = false;
-        llama_free_model(model);
+        llama_model_free(model);
 
         adjustNapiExternalMemorySubtract(Env(), loadedModelSize);
         loadedModelSize = 0;
@@ -426,7 +427,7 @@ Napi::Value AddonModel::Tokenize(const Napi::CallbackInfo& info) {
     std::string text = info[0].As<Napi::String>().Utf8Value();
     bool specialTokens = info[1].As<Napi::Boolean>().Value();
 
-    std::vector<llama_token> tokens = common_tokenize(model, text, false, specialTokens);
+    std::vector<llama_token> tokens = common_tokenize(vocab, text, false, specialTokens);
 
     Napi::Uint32Array result = Napi::Uint32Array::New(info.Env(), tokens.size());
     for (size_t i = 0; i < tokens.size(); ++i) {
@@ -449,10 +450,10 @@ Napi::Value AddonModel::Detokenize(const Napi::CallbackInfo& info) {
     std::string result;
     result.resize(std::max(result.capacity(), tokens.ElementLength()));
 
-    int n_chars = llama_detokenize(model, (llama_token*)tokens.Data(), tokens.ElementLength(), &result[0], result.size(), false, decodeSpecialTokens);
+    int n_chars = llama_detokenize(vocab, (llama_token*)tokens.Data(), tokens.ElementLength(), &result[0], result.size(), false, decodeSpecialTokens);
     if (n_chars < 0) {
         result.resize(-n_chars);
-        n_chars = llama_detokenize(model, (llama_token*)tokens.Data(), tokens.ElementLength(), &result[0], result.size(), false, decodeSpecialTokens);
+        n_chars = llama_detokenize(vocab, (llama_token*)tokens.Data(), tokens.ElementLength(), &result[0], result.size(), false, decodeSpecialTokens);
         GGML_ASSERT(n_chars <= result.size());  // whitespace trimming is performed after per-token detokenization
     }
 
@@ -467,7 +468,7 @@ Napi::Value AddonModel::GetTrainContextSize(const Napi::CallbackInfo& info) {
         return info.Env().Undefined();
     }
 
-    return Napi::Number::From(info.Env(), llama_n_ctx_train(model));
+    return Napi::Number::From(info.Env(), llama_model_n_ctx_train(model));
 }
 
 Napi::Value AddonModel::GetEmbeddingVectorSize(const Napi::CallbackInfo& info) {
@@ -476,7 +477,7 @@ Napi::Value AddonModel::GetEmbeddingVectorSize(const Napi::CallbackInfo& info) {
         return info.Env().Undefined();
     }
 
-    return Napi::Number::From(info.Env(), llama_n_embd(model));
+    return Napi::Number::From(info.Env(), llama_model_n_embd(model));
 }
 
 Napi::Value AddonModel::GetTotalSize(const Napi::CallbackInfo& info) {
@@ -515,7 +516,7 @@ Napi::Value AddonModel::TokenBos(const Napi::CallbackInfo& info) {
         return info.Env().Undefined();
     }
 
-    return getNapiControlToken(info, model, llama_token_bos(model));
+    return getNapiControlToken(info, vocab, llama_vocab_bos(vocab));
 }
 Napi::Value AddonModel::TokenEos(const Napi::CallbackInfo& info) {
     if (disposed) {
@@ -523,7 +524,7 @@ Napi::Value AddonModel::TokenEos(const Napi::CallbackInfo& info) {
         return info.Env().Undefined();
     }
 
-    return getNapiControlToken(info, model, llama_token_eos(model));
+    return getNapiControlToken(info, vocab, llama_vocab_eos(vocab));
 }
 Napi::Value AddonModel::TokenNl(const Napi::CallbackInfo& info) {
     if (disposed) {
@@ -531,7 +532,7 @@ Napi::Value AddonModel::TokenNl(const Napi::CallbackInfo& info) {
         return info.Env().Undefined();
     }
 
-    return getNapiToken(info, model, llama_token_nl(model));
+    return getNapiToken(info, vocab, llama_vocab_nl(vocab));
 }
 Napi::Value AddonModel::PrefixToken(const Napi::CallbackInfo& info) {
     if (disposed) {
@@ -539,7 +540,7 @@ Napi::Value AddonModel::PrefixToken(const Napi::CallbackInfo& info) {
         return info.Env().Undefined();
     }
 
-    return getNapiToken(info, model, llama_token_fim_pre(model));
+    return getNapiToken(info, vocab, llama_vocab_fim_pre(vocab));
 }
 Napi::Value AddonModel::MiddleToken(const Napi::CallbackInfo& info) {
     if (disposed) {
@@ -547,7 +548,7 @@ Napi::Value AddonModel::MiddleToken(const Napi::CallbackInfo& info) {
         return info.Env().Undefined();
     }
 
-    return getNapiToken(info, model, llama_token_fim_mid(model));
+    return getNapiToken(info, vocab, llama_vocab_fim_mid(vocab));
 }
 Napi::Value AddonModel::SuffixToken(const Napi::CallbackInfo& info) {
     if (disposed) {
@@ -555,7 +556,7 @@ Napi::Value AddonModel::SuffixToken(const Napi::CallbackInfo& info) {
         return info.Env().Undefined();
     }
 
-    return getNapiToken(info, model, llama_token_fim_suf(model));
+    return getNapiToken(info, vocab, llama_vocab_fim_suf(vocab));
 }
 Napi::Value AddonModel::EotToken(const Napi::CallbackInfo& info) {
     if (disposed) {
@@ -563,15 +564,7 @@ Napi::Value AddonModel::EotToken(const Napi::CallbackInfo& info) {
         return info.Env().Undefined();
     }
 
-    return getNapiToken(info, model, llama_token_eot(model));
-}
-Napi::Value AddonModel::ClsToken(const Napi::CallbackInfo& info) {
-    if (disposed) {
-        Napi::Error::New(info.Env(), "Model is disposed").ThrowAsJavaScriptException();
-        return info.Env().Undefined();
-    }
-
-    return getNapiToken(info, model, llama_token_cls(model));
+    return getNapiToken(info, vocab, llama_vocab_eot(vocab));
 }
 Napi::Value AddonModel::SepToken(const Napi::CallbackInfo& info) {
     if (disposed) {
@@ -579,7 +572,7 @@ Napi::Value AddonModel::SepToken(const Napi::CallbackInfo& info) {
         return info.Env().Undefined();
     }
 
-    return getNapiToken(info, model, llama_token_sep(model));
+    return getNapiToken(info, vocab, llama_vocab_sep(vocab));
 }
 Napi::Value AddonModel::GetTokenString(const Napi::CallbackInfo& info) {
     if (disposed) {
@@ -590,7 +583,7 @@ Napi::Value AddonModel::GetTokenString(const Napi::CallbackInfo& info) {
     int token = info[0].As<Napi::Number>().Int32Value();
     std::stringstream ss;
 
-    const char* str = llama_token_get_text(model, token);
+    const char* str = llama_vocab_get_text(vocab, token);
     if (str == nullptr) {
         return info.Env().Undefined();
     }
@@ -611,7 +604,7 @@ Napi::Value AddonModel::GetTokenAttributes(const Napi::CallbackInfo& info) {
     }
 
     int token = info[0].As<Napi::Number>().Int32Value();
-    auto tokenAttributes = llama_token_get_attr(model, token);
+    auto tokenAttributes = llama_vocab_get_attr(vocab, token);
 
     return Napi::Number::From(info.Env(), int32_t(tokenAttributes));
 }
@@ -627,7 +620,7 @@ Napi::Value AddonModel::IsEogToken(const Napi::CallbackInfo& info) {
 
     int token = info[0].As<Napi::Number>().Int32Value();
 
-    return Napi::Boolean::New(info.Env(), llama_token_is_eog(model, token));
+    return Napi::Boolean::New(info.Env(), llama_vocab_is_eog(vocab, token));
 }
 Napi::Value AddonModel::GetVocabularyType(const Napi::CallbackInfo& info) {
     if (disposed) {
@@ -635,17 +628,17 @@ Napi::Value AddonModel::GetVocabularyType(const Napi::CallbackInfo& info) {
         return info.Env().Undefined();
     }
 
-    auto vocabularyType = llama_vocab_type(model);
+    auto vocabularyType = llama_vocab_type(vocab);
 
     return Napi::Number::From(info.Env(), int32_t(vocabularyType));
 }
 Napi::Value AddonModel::ShouldPrependBosToken(const Napi::CallbackInfo& info) {
-    const bool addBos = llama_add_bos_token(model);
+    const bool addBos = llama_vocab_get_add_bos(vocab);
 
     return Napi::Boolean::New(info.Env(), addBos);
 }
 Napi::Value AddonModel::ShouldAppendEosToken(const Napi::CallbackInfo& info) {
-    const bool addEos = llama_add_eos_token(model);
+    const bool addEos = llama_vocab_get_add_eos(vocab);
 
     return Napi::Boolean::New(info.Env(), addEos);
 }
@@ -678,7 +671,6 @@ void AddonModel::init(Napi::Object exports) {
                 InstanceMethod("middleToken", &AddonModel::MiddleToken),
                 InstanceMethod("suffixToken", &AddonModel::SuffixToken),
                 InstanceMethod("eotToken", &AddonModel::EotToken),
-                InstanceMethod("clsToken", &AddonModel::ClsToken),
                 InstanceMethod("sepToken", &AddonModel::SepToken),
                 InstanceMethod("getTokenString", &AddonModel::GetTokenString),
                 InstanceMethod("getTokenAttributes", &AddonModel::GetTokenAttributes),

package/llama/addon/AddonModel.h

@@ -9,6 +9,7 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
         llama_model_params model_params;
         std::vector<llama_model_kv_override> kv_overrides;
         llama_model* model;
+        const llama_vocab* vocab;
         uint64_t loadedModelSize = 0;
         Napi::Reference<Napi::Object> addonExportsRef;
         bool hasAddonExportsRef = false;
@@ -49,7 +50,6 @@ class AddonModel : public Napi::ObjectWrap<AddonModel> {
         Napi::Value MiddleToken(const Napi::CallbackInfo& info);
         Napi::Value SuffixToken(const Napi::CallbackInfo& info);
         Napi::Value EotToken(const Napi::CallbackInfo& info);
-        Napi::Value ClsToken(const Napi::CallbackInfo& info);
         Napi::Value SepToken(const Napi::CallbackInfo& info);
         Napi::Value GetTokenString(const Napi::CallbackInfo& info);