@fugood/llama.node 0.3.2 → 0.3.4

package/CMakeLists.txt

@@ -6,6 +6,11 @@ project (llama-node)
 
 set(CMAKE_CXX_STANDARD 17)
 
+execute_process(COMMAND
+  git apply ${CMAKE_CURRENT_SOURCE_DIR}/scripts/ggml-cpu-CMakeLists.txt.patch
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+)
+
 if(NOT DEFINED napi_build_version)
   set(napi_build_version 6)
 endif()
@@ -62,6 +67,8 @@ if (VULKAN_SDK)
   find_package(Vulkan REQUIRED)
 endif()
 
+set(LLAMA_BUILD_COMMON ON CACHE BOOL "Build common")
+
 set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libraries")
 add_subdirectory("src/llama.cpp")
 

package/lib/binding.ts

@@ -8,6 +8,8 @@ export type ChatMessage = {
 export type LlamaModelOptions = {
   model: string
   embedding?: boolean
+  embd_normalize?: number
+  pooling_type?: number
   n_ctx?: number
   n_batch?: number
   n_threads?: number
@@ -23,7 +25,21 @@ export type LlamaCompletionOptions = {
   temperature?: number
   top_k?: number
   top_p?: number
-  repetition_penalty?: number
+  min_p?: number
+  mirostat?: number
+  mirostat_tau?: number
+  mirostat_eta?: number
+  penalty_last_n?: number
+  penalty_repeat?: number
+  penalty_freq?: number
+  penalty_present?: number
+  typ_p?: number
+  xtc_threshold?: number
+  xtc_probability?: number
+  dry_multiplier?: number
+  dry_base?: number
+  dry_allowed_length?: number
+  dry_penalty_last_n?: number
   n_predict?: number
   max_length?: number
   max_tokens?: number
@@ -54,6 +70,7 @@ export type EmbeddingResult = {
 export interface LlamaContext {
   new (options: LlamaModelOptions): LlamaContext
   getSystemInfo(): string
+  getModelInfo(): object
   getFormattedChat(messages: ChatMessage[]): string
   completion(options: LlamaCompletionOptions, callback?: (token: LlamaCompletionToken) => void): Promise<LlamaCompletionResult>
   stopCompletion(): void

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "description": "Llama.cpp for Node.js",
   "main": "lib/index.js",
   "scripts": {

package/src/DetokenizeWorker.cpp

@@ -8,7 +8,7 @@ DetokenizeWorker::DetokenizeWorker(const Napi::CallbackInfo &info,
       _tokens(std::move(tokens)) {}
 
 void DetokenizeWorker::Execute() {
-  const auto text = ::llama_detokenize(_sess->context(), _tokens);
+  const auto text = ::common_detokenize(_sess->context(), _tokens);
   _text = std::move(text);
 }
 

package/src/EmbeddingWorker.cpp

@@ -2,32 +2,42 @@
 #include "LlamaContext.h"
 
 EmbeddingWorker::EmbeddingWorker(const Napi::CallbackInfo &info,
-                                 LlamaSessionPtr &sess, std::string text)
-    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text) {}
+                                 LlamaSessionPtr &sess, std::string text, common_params &params)
+    : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text), _params(params) {}
 
 void EmbeddingWorker::Execute() {
   llama_kv_cache_clear(_sess->context());
-  auto tokens = ::llama_tokenize(_sess->context(), _text, true);
+  auto tokens = ::common_tokenize(_sess->context(), _text, true);
   // add SEP if not present
   if (tokens.empty() || tokens.back() != llama_token_sep(_sess->model())) {
     tokens.push_back(llama_token_sep(_sess->model()));
   }
   const int n_embd = llama_n_embd(_sess->model());
   do {
+    auto ctx = _sess->context();
     int ret =
-        llama_decode(_sess->context(),
-                     llama_batch_get_one(tokens.data(), tokens.size(), 0, 0));
+        llama_decode(ctx,
+                     llama_batch_get_one(tokens.data(), tokens.size()));
     if (ret < 0) {
       SetError("Failed to inference, code: " + std::to_string(ret));
       break;
     }
-    const float *embd = llama_get_embeddings_seq(_sess->context(), 0);
+
+    float *embd;
+    const enum llama_pooling_type pooling_type = llama_pooling_type(ctx);
+    if (pooling_type == LLAMA_POOLING_TYPE_NONE) {
+      embd = llama_get_embeddings(ctx);
+    } else {
+      embd = llama_get_embeddings_seq(ctx, 0);
+    }
     if (embd == nullptr) {
       SetError("Failed to get embeddings");
       break;
     }
     _result.embedding.resize(n_embd);
-    memcpy(_result.embedding.data(), embd, n_embd * sizeof(float));
+    std::vector<float> embedding(embd, embd + n_embd), out(embd, embd + n_embd);
+        common_embd_normalize(embedding.data(), out.data(), n_embd, _params.embd_normalize);
+    memcpy(_result.embedding.data(), out.data(), n_embd * sizeof(float));
   } while (false);
 }
 

package/src/EmbeddingWorker.h

@@ -9,7 +9,7 @@ class EmbeddingWorker : public Napi::AsyncWorker,
                         public Napi::Promise::Deferred {
 public:
   EmbeddingWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
-                  std::string text);
+                  std::string text, common_params &params);
 
 protected:
   void Execute();
@@ -19,5 +19,6 @@ protected:
 private:
   LlamaSessionPtr _sess;
   std::string _text;
+  common_params _params;
   EmbeddingResult _result;
 };

package/src/LlamaCompletionWorker.cpp

@@ -34,7 +34,7 @@ size_t findStoppingStrings(const std::string &text,
 
 LlamaCompletionWorker::LlamaCompletionWorker(
     const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
-    Napi::Function callback, gpt_params params,
+    Napi::Function callback, common_params params,
     std::vector<std::string> stop_words)
     : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess),
       _params(params), _stop_words(stop_words) {
@@ -64,11 +64,11 @@ void LlamaCompletionWorker::Execute() {
 
   auto sparams = llama_sampler_chain_default_params();
 
-  LlamaCppSampling sampling{gpt_sampler_init(model, _params.sparams),
-                            gpt_sampler_free};
+  LlamaCppSampling sampling{common_sampler_init(model, _params.sampling),
+                            common_sampler_free};
 
   std::vector<llama_token> prompt_tokens =
-      ::llama_tokenize(ctx, _params.prompt, add_bos);
+      ::common_tokenize(ctx, _params.prompt, add_bos);
   n_input = prompt_tokens.size();
   if (_sess->tokens_ptr()->size() > 0) {
     n_cur = common_part(*(_sess->tokens_ptr()), prompt_tokens);
@@ -102,18 +102,18 @@ void LlamaCompletionWorker::Execute() {
       _result.truncated = true;
     }
     int ret = llama_decode(
-        ctx, llama_batch_get_one(embd->data() + n_cur, n_input, n_cur, 0));
+        ctx, llama_batch_get_one(embd->data() + n_cur, n_input));
     if (ret < 0) {
       SetError("Failed to decode token, code: " + std::to_string(ret));
       break;
     }
     // sample the next token
     const llama_token new_token_id =
-        gpt_sampler_sample(sampling.get(), ctx, -1);
-    gpt_sampler_accept(sampling.get(), new_token_id, true);
+        common_sampler_sample(sampling.get(), ctx, -1);
+    common_sampler_accept(sampling.get(), new_token_id, true);
     // prepare the next batch
     embd->emplace_back(new_token_id);
-    auto token = llama_token_to_piece(ctx, new_token_id);
+    auto token = common_token_to_piece(ctx, new_token_id);
     _result.text += token;
     n_cur += n_input;
     _result.tokens_evaluated += n_input;

package/src/LlamaCompletionWorker.h

@@ -12,7 +12,7 @@ class LlamaCompletionWorker : public Napi::AsyncWorker,
                               public Napi::Promise::Deferred {
 public:
   LlamaCompletionWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
-                        Napi::Function callback, gpt_params params,
+                        Napi::Function callback, common_params params,
                         std::vector<std::string> stop_words = {});
 
   ~LlamaCompletionWorker();
@@ -28,7 +28,7 @@ protected:
 
 private:
   LlamaSessionPtr _sess;
-  gpt_params _params;
+  common_params _params;
   std::vector<std::string> _stop_words;
   Napi::ThreadSafeFunction _tsfn;
   bool _has_callback = false;

package/src/LlamaContext.cpp

@@ -7,8 +7,8 @@
 #include "SaveSessionWorker.h"
 #include "TokenizeWorker.h"
 
-std::vector<llama_chat_msg> get_messages(Napi::Array messages) {
-  std::vector<llama_chat_msg> chat;
+std::vector<common_chat_msg> get_messages(Napi::Array messages) {
+  std::vector<common_chat_msg> chat;
   for (size_t i = 0; i < messages.Length(); i++) {
     auto message = messages.Get(i).As<Napi::Object>();
     chat.push_back({
@@ -25,6 +25,9 @@ void LlamaContext::Init(Napi::Env env, Napi::Object &exports) {
       {InstanceMethod<&LlamaContext::GetSystemInfo>(
            "getSystemInfo",
            static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::GetModelInfo>(
+           "getModelInfo",
+           static_cast<napi_property_attributes>(napi_enumerable)),
        InstanceMethod<&LlamaContext::GetFormattedChat>(
            "getFormattedChat",
            static_cast<napi_property_attributes>(napi_enumerable)),
@@ -67,14 +70,23 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
   }
   auto options = info[0].As<Napi::Object>();
 
-  gpt_params params;
+  common_params params;
   params.model = get_option<std::string>(options, "model", "");
   if (params.model.empty()) {
     Napi::TypeError::New(env, "Model is required").ThrowAsJavaScriptException();
   }
-  params.embedding = get_option<bool>(options, "embedding", false);
+
   params.n_ctx = get_option<int32_t>(options, "n_ctx", 512);
   params.n_batch = get_option<int32_t>(options, "n_batch", 2048);
+  params.embedding = get_option<bool>(options, "embedding", false);
+  if (params.embedding) {
+    // For non-causal models, batch size must be equal to ubatch size
+    params.n_ubatch = params.n_batch;
+  }
+  params.embd_normalize = get_option<int32_t>(options, "embd_normalize", 2);
+  int32_t pooling_type = get_option<int32_t>(options, "pooling_type", -1);
+  params.pooling_type = (enum llama_pooling_type) pooling_type;
+
   params.cpuparams.n_threads =
       get_option<int32_t>(options, "n_threads", cpu_get_num_math() / 2);
   params.n_gpu_layers = get_option<int32_t>(options, "n_gpu_layers", -1);
@@ -86,7 +98,7 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
   llama_backend_init();
   llama_numa_init(params.numa);
 
-  auto result = llama_init_from_gpt_params(params);
+  auto result = common_init_from_params(params);
 
   if (result.model == nullptr || result.context == nullptr) {
     Napi::TypeError::New(env, "Failed to load model")
@@ -94,7 +106,7 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
   }
 
   _sess = std::make_shared<LlamaSession>(result.model, result.context, params);
-  _info = gpt_params_get_system_info(params);
+  _info = common_params_get_system_info(params);
 }
 
 // getSystemInfo(): string
@@ -102,6 +114,44 @@ Napi::Value LlamaContext::GetSystemInfo(const Napi::CallbackInfo &info) {
   return Napi::String::New(info.Env(), _info);
 }
 
+bool validateModelChatTemplate(const struct llama_model * model) {
+    std::vector<char> model_template(2048, 0); // longest known template is about 1200 bytes
+    std::string template_key = "tokenizer.chat_template";
+    int32_t res = llama_model_meta_val_str(model, template_key.c_str(), model_template.data(), model_template.size());
+    if (res >= 0) {
+        llama_chat_message chat[] = {{"user", "test"}};
+        std::string tmpl = std::string(model_template.data(), model_template.size());
+        int32_t chat_res = llama_chat_apply_template(model, tmpl.c_str(), chat, 1, true, nullptr, 0);
+        return chat_res > 0;
+    }
+    return res > 0;
+}
+
+// getModelInfo(): object
+Napi::Value LlamaContext::GetModelInfo(const Napi::CallbackInfo &info) {
+  char desc[1024];
+  auto model = _sess->model();
+  llama_model_desc(model, desc, sizeof(desc));
+
+  int count = llama_model_meta_count(model);
+  Napi::Object metadata = Napi::Object::New(info.Env());
+  for (int i = 0; i < count; i++) {
+    char key[256];
+    llama_model_meta_key_by_index(model, i, key, sizeof(key));
+    char val[2048];
+    llama_model_meta_val_str_by_index(model, i, val, sizeof(val));
+
+    metadata.Set(key, val);
+  }
+  Napi::Object details = Napi::Object::New(info.Env());
+  details.Set("desc", desc);
+  details.Set("nParams", llama_model_n_params(model));
+  details.Set("size", llama_model_size(model));
+  details.Set("isChatTemplateSupported", validateModelChatTemplate(model));
+  details.Set("metadata", metadata);
+  return details;
+}
+
 // getFormattedChat(messages: [{ role: string, content: string }]): string
 Napi::Value LlamaContext::GetFormattedChat(const Napi::CallbackInfo &info) {
   Napi::Env env = info.Env();
@@ -109,7 +159,7 @@ Napi::Value LlamaContext::GetFormattedChat(const Napi::CallbackInfo &info) {
     Napi::TypeError::New(env, "Array expected").ThrowAsJavaScriptException();
   }
   auto messages = info[0].As<Napi::Array>();
-  auto formatted = llama_chat_apply_template(_sess->model(), "", get_messages(messages), true);
+  auto formatted = common_chat_apply_template(_sess->model(), "", get_messages(messages), true);
   return Napi::String::New(env, formatted);
 }
 
@@ -133,10 +183,10 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
   }
   auto options = info[0].As<Napi::Object>();
 
-  gpt_params params = _sess->params();
+  common_params params = _sess->params();
   if (options.Has("messages") && options.Get("messages").IsArray()) {
     auto messages = options.Get("messages").As<Napi::Array>();
-    auto formatted = llama_chat_apply_template(_sess->model(), "", get_messages(messages), true);
+    auto formatted = common_chat_apply_template(_sess->model(), "", get_messages(messages), true);
     params.prompt = formatted;
   } else {
     params.prompt = get_option<std::string>(options, "prompt", "");
@@ -146,30 +196,34 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
         .ThrowAsJavaScriptException();
   }
   params.n_predict = get_option<int32_t>(options, "n_predict", -1);
-  params.sparams.temp = get_option<float>(options, "temperature", 0.80f);
-  params.sparams.top_k = get_option<int32_t>(options, "top_k", 40);
-  params.sparams.top_p = get_option<float>(options, "top_p", 0.95f);
-  params.sparams.min_p = get_option<float>(options, "min_p", 0.05f);
-  params.sparams.tfs_z = get_option<float>(options, "tfs_z", 1.00f);
-  params.sparams.mirostat = get_option<int32_t>(options, "mirostat", 0.00f);
-  params.sparams.mirostat_tau =
+  params.sampling.temp = get_option<float>(options, "temperature", 0.80f);
+  params.sampling.top_k = get_option<int32_t>(options, "top_k", 40);
+  params.sampling.top_p = get_option<float>(options, "top_p", 0.95f);
+  params.sampling.min_p = get_option<float>(options, "min_p", 0.05f);
+  params.sampling.mirostat = get_option<int32_t>(options, "mirostat", 0.00f);
+  params.sampling.mirostat_tau =
       get_option<float>(options, "mirostat_tau", 5.00f);
-  params.sparams.mirostat_eta =
+  params.sampling.mirostat_eta =
       get_option<float>(options, "mirostat_eta", 0.10f);
-  params.sparams.penalty_last_n =
+  params.sampling.penalty_last_n =
       get_option<int32_t>(options, "penalty_last_n", 64);
-  params.sparams.penalty_repeat =
+  params.sampling.penalty_repeat =
       get_option<float>(options, "penalty_repeat", 1.00f);
-  params.sparams.penalty_freq =
+  params.sampling.penalty_freq =
       get_option<float>(options, "penalty_freq", 0.00f);
-  params.sparams.penalty_present =
+  params.sampling.penalty_present =
       get_option<float>(options, "penalty_present", 0.00f);
-  params.sparams.penalize_nl = get_option<bool>(options, "penalize_nl", false);
-  params.sparams.typ_p = get_option<float>(options, "typical_p", 1.00f);
-  params.sparams.ignore_eos = get_option<float>(options, "ignore_eos", false);
-  params.sparams.grammar = get_option<std::string>(options, "grammar", "");
+  params.sampling.typ_p = get_option<float>(options, "typical_p", 1.00f);
+  params.sampling.xtc_threshold = get_option<float>(options, "xtc_threshold", 0.00f);
+  params.sampling.xtc_probability = get_option<float>(options, "xtc_probability", 0.10f);
+  params.sampling.dry_multiplier = get_option<float>(options, "dry_multiplier", 1.75f);
+  params.sampling.dry_base = get_option<float>(options, "dry_base", 2);
+  params.sampling.dry_allowed_length = get_option<float>(options, "dry_allowed_length", -1);
+  params.sampling.dry_penalty_last_n = get_option<float>(options, "dry_penalty_last_n", 0);
+  params.sampling.ignore_eos = get_option<bool>(options, "ignore_eos", false);
+  params.sampling.grammar = get_option<std::string>(options, "grammar", "");
   params.n_keep = get_option<int32_t>(options, "n_keep", 0);
-  params.sparams.seed = get_option<int32_t>(options, "seed", LLAMA_DEFAULT_SEED);
+  params.sampling.seed = get_option<int32_t>(options, "seed", LLAMA_DEFAULT_SEED);
   std::vector<std::string> stop_words;
   if (options.Has("stop") && options.Get("stop").IsArray()) {
     auto stop_words_array = options.Get("stop").As<Napi::Array>();
@@ -244,8 +298,16 @@ Napi::Value LlamaContext::Embedding(const Napi::CallbackInfo &info) {
     Napi::TypeError::New(env, "Context is disposed")
         .ThrowAsJavaScriptException();
   }
+  auto options = Napi::Object::New(env);
+  if (info.Length() >= 2 && info[1].IsObject()) {
+    options = info[1].As<Napi::Object>();
+  }
+
+  common_params embdParams;
+  embdParams.embedding = true;
+  embdParams.embd_normalize = get_option<int32_t>(options, "embd_normalize", 2);
   auto text = info[0].ToString().Utf8Value();
-  auto *worker = new EmbeddingWorker(info, _sess, text);
+  auto *worker = new EmbeddingWorker(info, _sess, text, embdParams);
   worker->Queue();
   return worker->Promise();
 }

package/src/LlamaContext.h

@@ -9,6 +9,7 @@ public:
 
 private:
   Napi::Value GetSystemInfo(const Napi::CallbackInfo &info);
+  Napi::Value GetModelInfo(const Napi::CallbackInfo &info);
   Napi::Value GetFormattedChat(const Napi::CallbackInfo &info);
   Napi::Value Completion(const Napi::CallbackInfo &info);
   void StopCompletion(const Napi::CallbackInfo &info);
@@ -20,6 +21,7 @@ private:
   Napi::Value Release(const Napi::CallbackInfo &info);
 
   std::string _info;
+  Napi::Object _meta;
   LlamaSessionPtr _sess = nullptr;
   LlamaCompletionWorker *_wip = nullptr;
 };

package/src/TokenizeWorker.cpp

@@ -6,7 +6,7 @@ TokenizeWorker::TokenizeWorker(const Napi::CallbackInfo &info,
     : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text) {}
 
 void TokenizeWorker::Execute() {
-  const auto tokens = ::llama_tokenize(_sess->context(), _text, false);
+  const auto tokens = ::common_tokenize(_sess->context(), _text, false);
   _result.tokens = std::move(tokens);
 }
 

package/src/common.hpp

@@ -13,7 +13,7 @@
 
 typedef std::unique_ptr<llama_model, decltype(&llama_free_model)> LlamaCppModel;
 typedef std::unique_ptr<llama_context, decltype(&llama_free)> LlamaCppContext;
-typedef std::unique_ptr<gpt_sampler, decltype(&gpt_sampler_free)>
+typedef std::unique_ptr<common_sampler, decltype(&common_sampler_free)>
     LlamaCppSampling;
 typedef std::unique_ptr<llama_batch, decltype(&llama_batch_free)> LlamaCppBatch;
 
@@ -47,7 +47,7 @@ constexpr T get_option(const Napi::Object &options, const std::string &name,
 
 class LlamaSession {
 public:
-  LlamaSession(llama_model *model, llama_context *ctx, gpt_params params)
+  LlamaSession(llama_model *model, llama_context *ctx, common_params params)
       : model_(LlamaCppModel(model, llama_free_model)),
         ctx_(LlamaCppContext(ctx, llama_free)), params_(params) {
     tokens_.reserve(params.n_ctx);
@@ -65,7 +65,7 @@ public:
     tokens_ = std::move(tokens);
   }
 
-  inline const gpt_params &params() const { return params_; }
+  inline const common_params &params() const { return params_; }
 
   inline std::mutex &get_mutex() { return mutex; }
 
@@ -79,7 +79,7 @@ public:
 private:
   LlamaCppModel model_;
   LlamaCppContext ctx_;
-  const gpt_params params_;
+  const common_params params_;
   std::vector<llama_token> tokens_{};
   std::mutex mutex;
 };