@fugood/llama.node 0.1.0 → 0.2.1

package/CMakeLists.txt

@@ -64,6 +64,15 @@ if (VULKAN_SDK)
   find_package(Vulkan REQUIRED)
 endif()
 
+find_program(PATCH patch REQUIRED)
+
+add_custom_target(
+  patch ALL
+  COMMAND ${PATCH} -p1 -N < ${CMAKE_SOURCE_DIR}/patches/llama.patch || true
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src/llama.cpp
+  COMMENT "Applying patches"
+)
+
 set(LLAMA_STATIC ON CACHE BOOL "Build llama as static library")
 add_subdirectory("src/llama.cpp")
 
@@ -77,6 +86,12 @@ file(
     "src/LlamaCompletionWorker.h"
     "src/LlamaContext.cpp"
     "src/LlamaContext.h"
+    "src/TokenizeWorker.cpp"
+    "src/TokenizeWorker.h"
+    "src/DetokenizeWorker.cpp"
+    "src/DetokenizeWorker.h"
+    "src/EmbeddingWorker.cpp"
+    "src/EmbeddingWorker.h"
     "src/LoadSessionWorker.cpp"
     "src/LoadSessionWorker.h"
     "src/SaveSessionWorker.cpp"

package/README.md

@@ -30,7 +30,7 @@ const context = await loadModel({
 })
 
 // Do completion
-const { text, timings } = await context.completion(
+const { text } = await context.completion(
   {
     prompt: 'This is a conversation between user and llama, a friendly chatbot. respond in simple markdown.\n\nUser: Hello!\nLlama:',
     n_predict: 100,
@@ -47,7 +47,8 @@ console.log('Result:', text)
 
 ## Lib Variants
 
-- [x] `default`: General usage, Supported GPU: Metal (macOS) and Vulkan (Linux / Windows)
+- [x] `default`: General usage, not support GPU except macOS (Metal)
+- [x] `vulkan`: Support GPU Vulkan (Windows/Linux), but some scenario might unstable
 
 ## License
 

package/lib/binding.ts

@@ -37,11 +37,22 @@ export type LlamaCompletionToken = {
   token: string
 }
 
+export type TokenizeResult = {
+  tokens: Int32Array
+}
+
+export type EmbeddingResult = {
+  embedding: Float32Array
+}
+
 export interface LlamaContext {
   new (options: LlamaModelOptions): LlamaContext
   getSystemInfo(): string
   completion(options: LlamaCompletionOptions, callback?: (token: LlamaCompletionToken) => void): Promise<LlamaCompletionResult>
   stopCompletion(): void
+  tokenize(text: string): Promise<TokenizeResult>
+  detokenize(tokens: number[]): Promise<string>
+  embedding(text: string): Promise<EmbeddingResult>
   saveSession(path: string): Promise<void>
   loadSession(path: string): Promise<void>
   release(): Promise<void>
@@ -51,7 +62,7 @@ export interface Module {
   LlamaContext: LlamaContext
 }
 
-export type LibVariant = 'default' | 'opencl'
+export type LibVariant = 'default' | 'vulkan'
 
 const setupEnv = (variant?: string) => {
   const postfix = variant ? `-${variant}` : ''

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.1.0",
+  "version": "0.2.1",
   "description": "Llama.cpp for Node.js",
   "main": "lib/index.js",
   "scripts": {
@@ -38,6 +38,7 @@
     ]
   },
   "files": [
+    "patches/*.patch",
     "bin/**/*",
     "src/**/*.{c,cc,cpp,h,hh,hpp,txt,cmake}",
     "lib/*.js",

package/patches/llama.patch

@@ -0,0 +1,22 @@
+diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp
+index b9449be0..cfa0f774 100644
+--- a/ggml-vulkan.cpp
++++ b/ggml-vulkan.cpp
+@@ -525,9 +525,15 @@ static void ggml_vk_create_pipeline(ggml_backend_vk_context * ctx, vk_pipeline&
+         vk::PipelineCreateFlags(),
+         pipeline_shader_create_info,
+         pipeline->layout);
+-    pipeline->pipeline = ctx->device->device.createComputePipeline(VK_NULL_HANDLE, compute_pipeline_create_info).value;
+ 
+-    ctx->device->pipelines.push_back(pipeline);
++    try {
++        pipeline->pipeline = ctx->device->device.createComputePipeline(VK_NULL_HANDLE, compute_pipeline_create_info).value;
++        ctx->device->pipelines.push_back(pipeline);
++    } catch (vk::UnknownError const&) {
++        std::cerr << "ggml_vk_create_pipeline: Failed to create pipeline " << name << std::endl;
++        ggml_vk_destroy_pipeline(ctx->device->device, pipeline);
++        pipeline.reset();
++    }
+ }
+ 
+ static void ggml_vk_destroy_pipeline(vk::Device& device, vk_pipeline& pipeline) {

package/src/DetokenizeWorker.cpp

@@ -0,0 +1,22 @@
+#include "DetokenizeWorker.h"
+#include "LlamaContext.h"
+
+DetokenizeWorker::DetokenizeWorker(const Napi::CallbackInfo &info,
+                                   LlamaSessionPtr &sess,
+                                   std::vector<llama_token> &tokens)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess),
+      _tokens(std::move(tokens)) {}
+
+void DetokenizeWorker::Execute() {
+  const auto text = ::llama_detokenize_bpe(_sess->context(), _tokens);
+  _text = std::move(text);
+}
+
+void DetokenizeWorker::OnOK() {
+  Napi::Promise::Deferred::Resolve(
+      Napi::String::New(Napi::AsyncWorker::Env(), _text));
+}
+
+void DetokenizeWorker::OnError(const Napi::Error &err) {
+  Napi::Promise::Deferred::Reject(err.Value());
+}

package/src/DetokenizeWorker.h

@@ -0,0 +1,19 @@
+#include "common.hpp"
+#include <vector>
+
+class DetokenizeWorker : public Napi::AsyncWorker,
+                         public Napi::Promise::Deferred {
+public:
+  DetokenizeWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
+                   std::vector<llama_token> &tokens);
+
+protected:
+  void Execute();
+  void OnOK();
+  void OnError(const Napi::Error &err);
+
+private:
+  LlamaSessionPtr _sess;
+  std::vector<llama_token> _tokens;
+  std::string _text;
+};

package/src/EmbeddingWorker.cpp

@@ -0,0 +1,46 @@
+#include "EmbeddingWorker.h"
+#include "LlamaContext.h"
+
+EmbeddingWorker::EmbeddingWorker(const Napi::CallbackInfo &info,
+                                 LlamaSessionPtr &sess, std::string text)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text) {}
+
+void EmbeddingWorker::Execute() {
+  llama_kv_cache_clear(_sess->context());
+  auto tokens = ::llama_tokenize(_sess->context(), _text, true);
+  // add SEP if not present
+  if (tokens.empty() || tokens.back() != llama_token_sep(_sess->model())) {
+    tokens.push_back(llama_token_sep(_sess->model()));
+  }
+  const int n_embd = llama_n_embd(_sess->model());
+  do {
+    int ret =
+        llama_decode(_sess->context(),
+                     llama_batch_get_one(tokens.data(), tokens.size(), 0, 0));
+    if (ret < 0) {
+      SetError("Failed to inference, code: " + std::to_string(ret));
+      break;
+    }
+    const float *embd = llama_get_embeddings_seq(_sess->context(), 0);
+    if (embd == nullptr) {
+      SetError("Failed to get embeddings");
+      break;
+    }
+    _result.embedding.resize(n_embd);
+    memcpy(_result.embedding.data(), embd, n_embd * sizeof(float));
+  } while (false);
+}
+
+void EmbeddingWorker::OnOK() {
+  auto result = Napi::Object::New(Napi::AsyncWorker::Env());
+  auto embedding = Napi::Float32Array::New(Napi::AsyncWorker::Env(),
+                                           _result.embedding.size());
+  memcpy(embedding.Data(), _result.embedding.data(),
+         _result.embedding.size() * sizeof(float));
+  result.Set("embedding", embedding);
+  Napi::Promise::Deferred::Resolve(result);
+}
+
+void EmbeddingWorker::OnError(const Napi::Error &err) {
+  Napi::Promise::Deferred::Reject(err.Value());
+}

package/src/EmbeddingWorker.h

@@ -0,0 +1,23 @@
+#include "common.hpp"
+#include <vector>
+
+struct EmbeddingResult {
+  std::vector<float> embedding;
+};
+
+class EmbeddingWorker : public Napi::AsyncWorker,
+                        public Napi::Promise::Deferred {
+public:
+  EmbeddingWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
+                  std::string text);
+
+protected:
+  void Execute();
+  void OnOK();
+  void OnError(const Napi::Error &err);
+
+private:
+  LlamaSessionPtr _sess;
+  std::string _text;
+  EmbeddingResult _result;
+};

package/src/LlamaContext.cpp

@@ -1,8 +1,11 @@
 #include "LlamaContext.h"
+#include "DetokenizeWorker.h"
 #include "DisposeWorker.h"
+#include "EmbeddingWorker.h"
 #include "LlamaCompletionWorker.h"
 #include "LoadSessionWorker.h"
 #include "SaveSessionWorker.h"
+#include "TokenizeWorker.h"
 
 void LlamaContext::Init(Napi::Env env, Napi::Object &exports) {
   Napi::Function func = DefineClass(
@@ -16,6 +19,13 @@ void LlamaContext::Init(Napi::Env env, Napi::Object &exports) {
        InstanceMethod<&LlamaContext::StopCompletion>(
            "stopCompletion",
            static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::Tokenize>(
+           "tokenize", static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::Detokenize>(
+           "detokenize",
+           static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::Embedding>(
+           "embedding", static_cast<napi_property_attributes>(napi_enumerable)),
        InstanceMethod<&LlamaContext::SaveSession>(
            "saveSession",
            static_cast<napi_property_attributes>(napi_enumerable)),
@@ -158,6 +168,58 @@ void LlamaContext::StopCompletion(const Napi::CallbackInfo &info) {
   }
 }
 
+// tokenize(text: string): Promise<TokenizeResult>
+Napi::Value LlamaContext::Tokenize(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (info.Length() < 1 || !info[0].IsString()) {
+    Napi::TypeError::New(env, "String expected").ThrowAsJavaScriptException();
+  }
+  if (_sess == nullptr) {
+    Napi::TypeError::New(env, "Context is disposed")
+        .ThrowAsJavaScriptException();
+  }
+  auto text = info[0].ToString().Utf8Value();
+  auto *worker = new TokenizeWorker(info, _sess, text);
+  worker->Queue();
+  return worker->Promise();
+}
+
+// detokenize(tokens: number[]): Promise<string>
+Napi::Value LlamaContext::Detokenize(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (info.Length() < 1 || !info[0].IsArray()) {
+    Napi::TypeError::New(env, "Array expected").ThrowAsJavaScriptException();
+  }
+  if (_sess == nullptr) {
+    Napi::TypeError::New(env, "Context is disposed")
+        .ThrowAsJavaScriptException();
+  }
+  auto tokens = info[0].As<Napi::Array>();
+  std::vector<int32_t> token_ids;
+  for (size_t i = 0; i < tokens.Length(); i++) {
+    token_ids.push_back(tokens.Get(i).ToNumber().Int32Value());
+  }
+  auto *worker = new DetokenizeWorker(info, _sess, token_ids);
+  worker->Queue();
+  return worker->Promise();
+}
+
+// embedding(text: string): Promise<EmbeddingResult>
+Napi::Value LlamaContext::Embedding(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  if (info.Length() < 1 || !info[0].IsString()) {
+    Napi::TypeError::New(env, "String expected").ThrowAsJavaScriptException();
+  }
+  if (_sess == nullptr) {
+    Napi::TypeError::New(env, "Context is disposed")
+        .ThrowAsJavaScriptException();
+  }
+  auto text = info[0].ToString().Utf8Value();
+  auto *worker = new EmbeddingWorker(info, _sess, text);
+  worker->Queue();
+  return worker->Promise();
+}
+
 // saveSession(path: string): Promise<void> throws error
 Napi::Value LlamaContext::SaveSession(const Napi::CallbackInfo &info) {
   Napi::Env env = info.Env();

package/src/LlamaContext.h

@@ -11,6 +11,9 @@ private:
   Napi::Value GetSystemInfo(const Napi::CallbackInfo &info);
   Napi::Value Completion(const Napi::CallbackInfo &info);
   void StopCompletion(const Napi::CallbackInfo &info);
+  Napi::Value Tokenize(const Napi::CallbackInfo &info);
+  Napi::Value Detokenize(const Napi::CallbackInfo &info);
+  Napi::Value Embedding(const Napi::CallbackInfo &info);
   Napi::Value SaveSession(const Napi::CallbackInfo &info);
   Napi::Value LoadSession(const Napi::CallbackInfo &info);
   Napi::Value Release(const Napi::CallbackInfo &info);

package/src/TokenizeWorker.cpp

@@ -0,0 +1,26 @@
+#include "TokenizeWorker.h"
+#include "LlamaContext.h"
+
+TokenizeWorker::TokenizeWorker(const Napi::CallbackInfo &info,
+                               LlamaSessionPtr &sess, std::string text)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text) {}
+
+void TokenizeWorker::Execute() {
+  const auto tokens = ::llama_tokenize(_sess->context(), _text, false);
+  _result.tokens = std::move(tokens);
+}
+
+void TokenizeWorker::OnOK() {
+  Napi::HandleScope scope(Napi::AsyncWorker::Env());
+  auto result = Napi::Object::New(Napi::AsyncWorker::Env());
+  auto tokens =
+      Napi::Int32Array::New(Napi::AsyncWorker::Env(), _result.tokens.size());
+  memcpy(tokens.Data(), _result.tokens.data(),
+         _result.tokens.size() * sizeof(llama_token));
+  result.Set("tokens", tokens);
+  Napi::Promise::Deferred::Resolve(result);
+}
+
+void TokenizeWorker::OnError(const Napi::Error &err) {
+  Napi::Promise::Deferred::Reject(err.Value());
+}

package/src/TokenizeWorker.h

@@ -0,0 +1,23 @@
+#include "common.hpp"
+#include <vector>
+
+struct TokenizeResult {
+  std::vector<llama_token> tokens;
+};
+
+class TokenizeWorker : public Napi::AsyncWorker,
+                       public Napi::Promise::Deferred {
+public:
+  TokenizeWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
+                 std::string text);
+
+protected:
+  void Execute();
+  void OnOK();
+  void OnError(const Napi::Error &err);
+
+private:
+  LlamaSessionPtr _sess;
+  std::string _text;
+  TokenizeResult _result;
+};

package/src/common.hpp

@@ -47,7 +47,8 @@ constexpr T get_option(const Napi::Object &options, const std::string &name,
 class LlamaSession {
 public:
   LlamaSession(llama_model *model, llama_context *ctx, gpt_params params)
-      : model_(LlamaCppModel(model, llama_free_model)), ctx_(LlamaCppContext(ctx, llama_free)), params_(params) {
+      : model_(LlamaCppModel(model, llama_free_model)),
+        ctx_(LlamaCppContext(ctx, llama_free)), params_(params) {
     tokens_.reserve(params.n_ctx);
   }
 
@@ -57,7 +58,7 @@ public:
 
   inline llama_model *model() { return model_.get(); }
 
-  inline std::vector<llama_token>* tokens_ptr() { return &tokens_; }
+  inline std::vector<llama_token> *tokens_ptr() { return &tokens_; }
 
   inline void set_tokens(std::vector<llama_token> tokens) {
     tokens_ = std::move(tokens);

package/src/llama.cpp/CMakeLists.txt

@@ -123,6 +123,7 @@ set(LLAMA_METAL_MACOSX_VERSION_MIN "" CACHE STRING
 set(LLAMA_METAL_STD "" CACHE STRING          "llama: metal standard version (-std flag)")
 option(LLAMA_KOMPUTE                         "llama: use Kompute"                               OFF)
 option(LLAMA_MPI                             "llama: use MPI"                                   OFF)
+option(LLAMA_RPC                             "llama: use RPC"                                   OFF)
 option(LLAMA_QKK_64                          "llama: use super-block size of 64 for k-quants"   OFF)
 option(LLAMA_SYCL                            "llama: use SYCL"                                  OFF)
 option(LLAMA_SYCL_F16                        "llama: use 16 bit floats for sycl calculations"   OFF)
@@ -296,7 +297,7 @@ if (LLAMA_BLAS)
     if (LLAMA_STATIC)
         set(BLA_STATIC ON)
     endif()
-    if ($(CMAKE_VERSION) VERSION_GREATER_EQUAL 3.22)
+    if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.22)
         set(BLA_SIZEOF_INTEGER 8)
     endif()
 
@@ -494,6 +495,17 @@ if (LLAMA_MPI)
     endif()
 endif()
 
+if (LLAMA_RPC)
+    add_compile_definitions(GGML_USE_RPC)
+
+    if (WIN32)
+        set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ws2_32)
+    endif()
+
+    set(GGML_HEADERS_RPC ggml-rpc.h)
+    set(GGML_SOURCES_RPC ggml-rpc.cpp)
+endif()
+
 if (LLAMA_CLBLAST)
     find_package(CLBlast)
     if (CLBlast_FOUND)
@@ -1176,6 +1188,7 @@ add_library(ggml OBJECT
             ${GGML_SOURCES_OPENCL}    ${GGML_HEADERS_OPENCL}
             ${GGML_SOURCES_METAL}     ${GGML_HEADERS_METAL}
             ${GGML_SOURCES_MPI}       ${GGML_HEADERS_MPI}
+            ${GGML_SOURCES_RPC}       ${GGML_HEADERS_RPC}
             ${GGML_SOURCES_EXTRA}     ${GGML_HEADERS_EXTRA}
             ${GGML_SOURCES_SYCL}      ${GGML_HEADERS_SYCL}
             ${GGML_SOURCES_KOMPUTE}   ${GGML_HEADERS_KOMPUTE}
@@ -1281,17 +1294,6 @@ install(
         WORLD_READ
         WORLD_EXECUTE
     DESTINATION ${CMAKE_INSTALL_BINDIR})
-install(
-    FILES convert-lora-to-ggml.py
-    PERMISSIONS
-        OWNER_READ
-        OWNER_WRITE
-        OWNER_EXECUTE
-        GROUP_READ
-        GROUP_EXECUTE
-        WORLD_READ
-        WORLD_EXECUTE
-    DESTINATION ${CMAKE_INSTALL_BINDIR})
 if (LLAMA_METAL)
     install(
         FILES ggml-metal.metal

package/src/llama.cpp/common/common.cpp

@@ -901,6 +901,10 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
         params.interactive = true;
         return true;
     }
+    if (arg == "--interactive-specials") {
+        params.interactive_specials = true;
+        return true;
+    }
     if (arg == "--embedding") {
         params.embedding = true;
         return true;
@@ -1056,6 +1060,14 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
 #endif // GGML_USE_CUDA_SYCL_VULKAN
         return true;
     }
+    if (arg == "--rpc") {
+        if (++i >= argc) {
+            invalid_param = true;
+            return true;
+        }
+        params.rpc_servers = argv[i];
+        return true;
+    }
     if (arg == "--no-mmap") {
         params.use_mmap = false;
         return true;
@@ -1367,14 +1379,12 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {
         if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {
             std::replace(arg.begin(), arg.end(), '_', '-');
         }
-
         if (!gpt_params_find_arg(argc, argv, arg, params, i, invalid_param)) {
             throw std::invalid_argument("error: unknown argument: " + arg);
         }
-    }
-
-    if (invalid_param) {
-        throw std::invalid_argument("error: invalid parameter for argument: " + arg);
+        if (invalid_param) {
+            throw std::invalid_argument("error: invalid parameter for argument: " + arg);
+        }
     }
 
     if (params.prompt_cache_all &&
@@ -1422,6 +1432,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
     printf("  -h, --help            show this help message and exit\n");
     printf("  --version             show version and build info\n");
     printf("  -i, --interactive     run in interactive mode\n");
+    printf("  --interactive-specials allow special tokens in user text, in interactive mode\n");
     printf("  --interactive-first   run in interactive mode and wait for input right away\n");
     printf("  -cnv, --conversation  run in conversation mode (does not print special tokens and suffix/prefix)\n");
     printf("  -ins, --instruct      run in instruction mode (use with Alpaca models)\n");
@@ -1554,6 +1565,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
         printf("  -mg i, --main-gpu i   the GPU to use for the model (with split-mode = none),\n");
         printf("                        or for intermediate results and KV (with split-mode = row) (default: %d)\n", params.main_gpu);
     }
+    printf("  --rpc SERVERS         comma separated list of RPC servers\n");
     printf("  --verbose-prompt      print a verbose prompt before generation (default: %s)\n", params.verbose_prompt ? "true" : "false");
     printf("  --no-display-prompt   don't print prompt at generation (default: %s)\n", !params.display_prompt ? "true" : "false");
     printf("  -gan N, --grp-attn-n N\n");
@@ -1827,6 +1839,7 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params &
     if (params.n_gpu_layers != -1) {
         mparams.n_gpu_layers = params.n_gpu_layers;
     }
+    mparams.rpc_servers     = params.rpc_servers.c_str();
     mparams.main_gpu        = params.main_gpu;
     mparams.split_mode      = params.split_mode;
     mparams.tensor_split    = params.tensor_split;
@@ -2652,6 +2665,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const l
     dump_string_yaml_multiline(stream, "in_suffix", params.input_prefix.c_str());
     fprintf(stream, "instruct: %s # default: false\n", params.instruct ? "true" : "false");
     fprintf(stream, "interactive: %s # default: false\n", params.interactive ? "true" : "false");
+    fprintf(stream, "interactive_specials: %s # default: false\n", params.interactive_specials ? "true" : "false");
     fprintf(stream, "interactive_first: %s # default: false\n", params.interactive_first ? "true" : "false");
     fprintf(stream, "keep: %d # default: 0\n", params.n_keep);
     fprintf(stream, "logdir: %s # default: unset (no logging)\n", params.logdir.c_str());

package/src/llama.cpp/common/common.h

@@ -82,6 +82,7 @@ struct gpt_params {
     float   yarn_beta_slow        = 1.0f;  // YaRN high correction dim
     int32_t yarn_orig_ctx         = 0;     // YaRN original context length
     float   defrag_thold          = -1.0f; // KV cache defragmentation threshold
+    std::string rpc_servers       = "";    // comma separated list of RPC servers
 
     ggml_backend_sched_eval_callback cb_eval = nullptr;
     void * cb_eval_user_data                 = nullptr;
@@ -140,6 +141,7 @@ struct gpt_params {
     bool random_prompt     = false; // do not randomize prompt if none provided
     bool use_color         = false; // use color to distinguish generations and inputs
     bool interactive       = false; // interactive mode
+    bool interactive_specials = false; // whether to allow special tokens from user, during interactive mode
     bool conversation      = false; // conversation mode (does not print special tokens and suffix/prefix)
     bool chatml            = false; // chatml mode (used for models trained on chatml syntax)
     bool prompt_cache_all  = false; // save user input and generations to prompt cache

package/src/llama.cpp/common/grammar-parser.cpp

@@ -142,6 +142,9 @@ namespace grammar_parser {
                 pos++;
                 last_sym_start = out_elements.size();
                 while (*pos != '"') {
+                    if (!*pos) {
+                        throw std::runtime_error("unexpected end of input");
+                    }
                     auto char_pair = parse_char(pos);
                          pos       = char_pair.second;
                     out_elements.push_back({LLAMA_GRETYPE_CHAR, char_pair.first});
@@ -156,6 +159,9 @@ namespace grammar_parser {
                 }
                 last_sym_start = out_elements.size();
                 while (*pos != ']') {
+                    if (!*pos) {
+                        throw std::runtime_error("unexpected end of input");
+                    }
                     auto char_pair = parse_char(pos);
                          pos       = char_pair.second;
                     enum llama_gretype type = last_sym_start < out_elements.size()
@@ -164,6 +170,9 @@ namespace grammar_parser {
 
                     out_elements.push_back({type, char_pair.first});
                     if (pos[0] == '-' && pos[1] != ']') {
+                        if (!pos[1]) {
+                            throw std::runtime_error("unexpected end of input");
+                        }
                         auto endchar_pair = parse_char(pos + 1);
                              pos          = endchar_pair.second;
                         out_elements.push_back({LLAMA_GRETYPE_CHAR_RNG_UPPER, endchar_pair.first});

package/src/llama.cpp/common/sampling.cpp

@@ -35,7 +35,7 @@ struct llama_sampling_context * llama_sampling_init(const struct llama_sampling_
 
     result->prev.resize(params.n_prev);
 
-    result->n_considered = 0;
+    result->n_valid = 0;
 
     llama_sampling_set_rng_seed(result, params.seed);
 
@@ -66,7 +66,7 @@ void llama_sampling_reset(llama_sampling_context * ctx) {
 
     std::fill(ctx->prev.begin(), ctx->prev.end(), 0);
     ctx->cur.clear();
-    ctx->n_considered = 0;
+    ctx->n_valid = 0;
 }
 
 void llama_sampling_set_rng_seed(struct llama_sampling_context * ctx, uint32_t seed) {
@@ -256,7 +256,7 @@ static llama_token llama_sampling_sample_impl(
         }
     }
 
-    ctx_sampling->n_considered = cur_p.size;
+    ctx_sampling->n_valid = temp == 0.0f ? 0 : cur_p.size;
 
     return id;
 }

package/src/llama.cpp/common/sampling.h

@@ -81,7 +81,7 @@ struct llama_sampling_context {
     // TODO: replace with ring-buffer
     std::vector<llama_token>      prev;
     std::vector<llama_token_data> cur;
-    size_t n_considered;
+    size_t n_valid; // Number of correct top tokens with correct probabilities.
 
     std::mt19937 rng;
 };