@fugood/llama.node 0.3.2 → 0.3.3

package/CMakeLists.txt

@@ -62,6 +62,8 @@ if (VULKAN_SDK)
   find_package(Vulkan REQUIRED)
 endif()
 
+set(LLAMA_BUILD_COMMON ON CACHE BOOL "Build common")
+
 set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libraries")
 add_subdirectory("src/llama.cpp")
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.3.2",
+  "version": "0.3.3",
   "description": "Llama.cpp for Node.js",
   "main": "lib/index.js",
   "scripts": {

package/src/DetokenizeWorker.cpp

@@ -8,7 +8,7 @@ DetokenizeWorker::DetokenizeWorker(const Napi::CallbackInfo &info,
       _tokens(std::move(tokens)) {}
 
 void DetokenizeWorker::Execute() {
-  const auto text = ::llama_detokenize(_sess->context(), _tokens);
+  const auto text = ::common_detokenize(_sess->context(), _tokens);
   _text = std::move(text);
 }
 

package/src/EmbeddingWorker.cpp

@@ -7,7 +7,7 @@ EmbeddingWorker::EmbeddingWorker(const Napi::CallbackInfo &info,
 
 void EmbeddingWorker::Execute() {
   llama_kv_cache_clear(_sess->context());
-  auto tokens = ::llama_tokenize(_sess->context(), _text, true);
+  auto tokens = ::common_tokenize(_sess->context(), _text, true);
   // add SEP if not present
   if (tokens.empty() || tokens.back() != llama_token_sep(_sess->model())) {
     tokens.push_back(llama_token_sep(_sess->model()));
@@ -16,7 +16,7 @@ void EmbeddingWorker::Execute() {
   do {
     int ret =
         llama_decode(_sess->context(),
-                     llama_batch_get_one(tokens.data(), tokens.size(), 0, 0));
+                     llama_batch_get_one(tokens.data(), tokens.size()));
     if (ret < 0) {
       SetError("Failed to inference, code: " + std::to_string(ret));
       break;

package/src/LlamaCompletionWorker.cpp

@@ -34,7 +34,7 @@ size_t findStoppingStrings(const std::string &text,
 
 LlamaCompletionWorker::LlamaCompletionWorker(
     const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
-    Napi::Function callback, gpt_params params,
+    Napi::Function callback, common_params params,
     std::vector<std::string> stop_words)
     : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess),
       _params(params), _stop_words(stop_words) {
@@ -64,11 +64,11 @@ void LlamaCompletionWorker::Execute() {
 
   auto sparams = llama_sampler_chain_default_params();
 
-  LlamaCppSampling sampling{gpt_sampler_init(model, _params.sparams),
-                            gpt_sampler_free};
+  LlamaCppSampling sampling{common_sampler_init(model, _params.sparams),
+                            common_sampler_free};
 
   std::vector<llama_token> prompt_tokens =
-      ::llama_tokenize(ctx, _params.prompt, add_bos);
+      ::common_tokenize(ctx, _params.prompt, add_bos);
   n_input = prompt_tokens.size();
   if (_sess->tokens_ptr()->size() > 0) {
     n_cur = common_part(*(_sess->tokens_ptr()), prompt_tokens);
@@ -102,18 +102,18 @@ void LlamaCompletionWorker::Execute() {
       _result.truncated = true;
     }
     int ret = llama_decode(
-        ctx, llama_batch_get_one(embd->data() + n_cur, n_input, n_cur, 0));
+        ctx, llama_batch_get_one(embd->data() + n_cur, n_input));
     if (ret < 0) {
       SetError("Failed to decode token, code: " + std::to_string(ret));
       break;
     }
     // sample the next token
     const llama_token new_token_id =
-        gpt_sampler_sample(sampling.get(), ctx, -1);
-    gpt_sampler_accept(sampling.get(), new_token_id, true);
+        common_sampler_sample(sampling.get(), ctx, -1);
+    common_sampler_accept(sampling.get(), new_token_id, true);
     // prepare the next batch
     embd->emplace_back(new_token_id);
-    auto token = llama_token_to_piece(ctx, new_token_id);
+    auto token = common_token_to_piece(ctx, new_token_id);
     _result.text += token;
     n_cur += n_input;
     _result.tokens_evaluated += n_input;

package/src/LlamaCompletionWorker.h

@@ -12,7 +12,7 @@ class LlamaCompletionWorker : public Napi::AsyncWorker,
                               public Napi::Promise::Deferred {
 public:
   LlamaCompletionWorker(const Napi::CallbackInfo &info, LlamaSessionPtr &sess,
-                        Napi::Function callback, gpt_params params,
+                        Napi::Function callback, common_params params,
                         std::vector<std::string> stop_words = {});
 
   ~LlamaCompletionWorker();
@@ -28,7 +28,7 @@ protected:
 
 private:
   LlamaSessionPtr _sess;
-  gpt_params _params;
+  common_params _params;
   std::vector<std::string> _stop_words;
   Napi::ThreadSafeFunction _tsfn;
   bool _has_callback = false;

package/src/LlamaContext.cpp

@@ -7,8 +7,8 @@
 #include "SaveSessionWorker.h"
 #include "TokenizeWorker.h"
 
-std::vector<llama_chat_msg> get_messages(Napi::Array messages) {
-  std::vector<llama_chat_msg> chat;
+std::vector<common_chat_msg> get_messages(Napi::Array messages) {
+  std::vector<common_chat_msg> chat;
   for (size_t i = 0; i < messages.Length(); i++) {
     auto message = messages.Get(i).As<Napi::Object>();
     chat.push_back({
@@ -67,7 +67,7 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
   }
   auto options = info[0].As<Napi::Object>();
 
-  gpt_params params;
+  common_params params;
   params.model = get_option<std::string>(options, "model", "");
   if (params.model.empty()) {
     Napi::TypeError::New(env, "Model is required").ThrowAsJavaScriptException();
@@ -86,7 +86,7 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
   llama_backend_init();
   llama_numa_init(params.numa);
 
-  auto result = llama_init_from_gpt_params(params);
+  auto result = common_init_from_params(params);
 
   if (result.model == nullptr || result.context == nullptr) {
     Napi::TypeError::New(env, "Failed to load model")
@@ -94,7 +94,7 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
   }
 
   _sess = std::make_shared<LlamaSession>(result.model, result.context, params);
-  _info = gpt_params_get_system_info(params);
+  _info = common_params_get_system_info(params);
 }
 
 // getSystemInfo(): string
@@ -109,7 +109,7 @@ Napi::Value LlamaContext::GetFormattedChat(const Napi::CallbackInfo &info) {
     Napi::TypeError::New(env, "Array expected").ThrowAsJavaScriptException();
   }
   auto messages = info[0].As<Napi::Array>();
-  auto formatted = llama_chat_apply_template(_sess->model(), "", get_messages(messages), true);
+  auto formatted = common_chat_apply_template(_sess->model(), "", get_messages(messages), true);
   return Napi::String::New(env, formatted);
 }
 
@@ -133,10 +133,10 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
   }
   auto options = info[0].As<Napi::Object>();
 
-  gpt_params params = _sess->params();
+  common_params params = _sess->params();
   if (options.Has("messages") && options.Get("messages").IsArray()) {
     auto messages = options.Get("messages").As<Napi::Array>();
-    auto formatted = llama_chat_apply_template(_sess->model(), "", get_messages(messages), true);
+    auto formatted = common_chat_apply_template(_sess->model(), "", get_messages(messages), true);
     params.prompt = formatted;
   } else {
     params.prompt = get_option<std::string>(options, "prompt", "");
@@ -150,7 +150,6 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
   params.sparams.top_k = get_option<int32_t>(options, "top_k", 40);
   params.sparams.top_p = get_option<float>(options, "top_p", 0.95f);
   params.sparams.min_p = get_option<float>(options, "min_p", 0.05f);
-  params.sparams.tfs_z = get_option<float>(options, "tfs_z", 1.00f);
   params.sparams.mirostat = get_option<int32_t>(options, "mirostat", 0.00f);
   params.sparams.mirostat_tau =
       get_option<float>(options, "mirostat_tau", 5.00f);

package/src/TokenizeWorker.cpp

@@ -6,7 +6,7 @@ TokenizeWorker::TokenizeWorker(const Napi::CallbackInfo &info,
     : AsyncWorker(info.Env()), Deferred(info.Env()), _sess(sess), _text(text) {}
 
 void TokenizeWorker::Execute() {
-  const auto tokens = ::llama_tokenize(_sess->context(), _text, false);
+  const auto tokens = ::common_tokenize(_sess->context(), _text, false);
   _result.tokens = std::move(tokens);
 }
 

package/src/common.hpp

@@ -13,7 +13,7 @@
 
 typedef std::unique_ptr<llama_model, decltype(&llama_free_model)> LlamaCppModel;
 typedef std::unique_ptr<llama_context, decltype(&llama_free)> LlamaCppContext;
-typedef std::unique_ptr<gpt_sampler, decltype(&gpt_sampler_free)>
+typedef std::unique_ptr<common_sampler, decltype(&common_sampler_free)>
     LlamaCppSampling;
 typedef std::unique_ptr<llama_batch, decltype(&llama_batch_free)> LlamaCppBatch;
 
@@ -47,7 +47,7 @@ constexpr T get_option(const Napi::Object &options, const std::string &name,
 
 class LlamaSession {
 public:
-  LlamaSession(llama_model *model, llama_context *ctx, gpt_params params)
+  LlamaSession(llama_model *model, llama_context *ctx, common_params params)
       : model_(LlamaCppModel(model, llama_free_model)),
         ctx_(LlamaCppContext(ctx, llama_free)), params_(params) {
     tokens_.reserve(params.n_ctx);
@@ -65,7 +65,7 @@ public:
     tokens_ = std::move(tokens);
   }
 
-  inline const gpt_params &params() const { return params_; }
+  inline const common_params &params() const { return params_; }
 
   inline std::mutex &get_mutex() { return mutex; }
 
@@ -79,7 +79,7 @@ public:
 private:
   LlamaCppModel model_;
   LlamaCppContext ctx_;
-  const gpt_params params_;
+  const common_params params_;
   std::vector<llama_token> tokens_{};
   std::mutex mutex;
 };

package/src/llama.cpp/.github/workflows/build.yml

@@ -55,7 +55,13 @@ jobs:
           sysctl -a
           mkdir build
           cd build
-          cmake -DLLAMA_FATAL_WARNINGS=ON -DGGML_METAL_EMBED_LIBRARY=ON -DLLAMA_CURL=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF ..
+          cmake .. \
+            -DLLAMA_FATAL_WARNINGS=ON \
+            -DLLAMA_CURL=ON \
+            -DGGML_METAL_USE_BF16=ON \
+            -DGGML_METAL_EMBED_LIBRARY=ON \
+            -DGGML_RPC=ON \
+            -DBUILD_SHARED_LIBS=OFF
           cmake --build . --config Release -j $(sysctl -n hw.logicalcpu)
 
       - name: Test
@@ -92,7 +98,7 @@ jobs:
           name: llama-bin-macos-arm64.zip
 
   macOS-latest-cmake-x64:
-    runs-on: macos-12
+    runs-on: macos-13
 
     steps:
       - name: Clone
@@ -113,7 +119,12 @@ jobs:
           sysctl -a
           # Metal is disabled due to intermittent failures with Github runners not having a GPU:
           # https://github.com/ggerganov/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
-          cmake -B build -DLLAMA_FATAL_WARNINGS=ON -DGGML_METAL=OFF -DLLAMA_CURL=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF
+          cmake -B build \
+            -DLLAMA_FATAL_WARNINGS=ON \
+            -DLLAMA_CURL=ON \
+            -DGGML_METAL=OFF \
+            -DGGML_RPC=ON \
+            -DBUILD_SHARED_LIBS=OFF
           cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
 
       - name: Test
@@ -394,15 +405,36 @@ jobs:
       - name: Build with native CMake HIP support
         id: cmake_build
         run: |
-          cmake -B build -S . -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" -DGGML_HIPBLAS=ON
+          cmake -B build -S . -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" -DGGML_HIP=ON
           cmake --build build --config Release -j $(nproc)
 
       - name: Build with legacy HIP support
         id: cmake_build_legacy_hip
         run: |
-          cmake -B build2 -S . -DCMAKE_C_COMPILER=hipcc -DCMAKE_CXX_COMPILER=hipcc -DGGML_HIPBLAS=ON
+          cmake -B build2 -S . -DCMAKE_C_COMPILER=hipcc -DCMAKE_CXX_COMPILER=hipcc -DGGML_HIP=ON
           cmake --build build2 --config Release -j $(nproc)
 
+  ubuntu-22-cmake-musa:
+    runs-on: ubuntu-22.04
+    container: mthreads/musa:rc3.1.0-devel-ubuntu22.04
+
+    steps:
+      - name: Clone
+        id: checkout
+        uses: actions/checkout@v4
+
+      - name: Dependencies
+        id: depends
+        run: |
+          apt-get update
+          apt-get install -y build-essential git cmake libcurl4-openssl-dev
+
+      - name: Build with native CMake MUSA support
+        id: cmake_build
+        run: |
+          cmake -B build -S . -DGGML_MUSA=ON
+          cmake --build build --config Release -j $(nproc)
+
   ubuntu-22-cmake-sycl:
     runs-on: ubuntu-22.04
 
@@ -569,6 +601,7 @@ jobs:
           mkdir build
           cd build
           cmake -G Xcode .. \
+            -DGGML_METAL_USE_BF16=ON \
             -DGGML_METAL_EMBED_LIBRARY=ON \
             -DLLAMA_BUILD_EXAMPLES=OFF \
             -DLLAMA_BUILD_TESTS=OFF \
@@ -599,6 +632,7 @@ jobs:
           mkdir build
           cd build
           cmake -G Xcode .. \
+            -DGGML_METAL_USE_BF16=ON \
             -DGGML_METAL_EMBED_LIBRARY=ON \
             -DLLAMA_BUILD_EXAMPLES=OFF \
             -DLLAMA_BUILD_TESTS=OFF \
@@ -734,7 +768,7 @@ jobs:
         id: clone_kompute
         if: ${{ matrix.build == 'kompute-x64' }}
         run: |
-          git submodule update --init ggml/src/kompute
+          git submodule update --init ggml/src/ggml-kompute/kompute
 
       - name: Download OpenBLAS
         id: get_openblas
@@ -917,7 +951,7 @@ jobs:
         shell: bash
 
     env:
-      WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/7dff44ba-e3af-4448-841c-0d616c8da6e7/w_BaseKit_p_2024.1.0.595_offline.exe
+      WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b380d914-366b-4b77-a74a-05e3c38b3514/intel-oneapi-base-toolkit-2025.0.0.882_offline.exe
       WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel
       ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
     steps:
@@ -1001,7 +1035,7 @@ jobs:
         run: |
           $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path)
           $env:CMAKE_PREFIX_PATH="${env:HIP_PATH}"
-          cmake -G "Unix Makefiles" -B build -S . -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" -DGGML_HIPBLAS=ON -DCMAKE_BUILD_TYPE=Release -DGGML_RPC=ON
+          cmake -G "Unix Makefiles" -B build -S . -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" -DGGML_HIP=ON -DCMAKE_BUILD_TYPE=Release -DGGML_RPC=ON
           cmake --build build -j ${env:NUMBER_OF_PROCESSORS}
 
   windows-latest-cmake-hip-release:
@@ -1037,7 +1071,7 @@ jobs:
         run: |
           $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path)
           $env:CMAKE_PREFIX_PATH="${env:HIP_PATH}"
-          cmake -G "Unix Makefiles" -B build -S . -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" -DGGML_HIPBLAS=ON -DCMAKE_BUILD_TYPE=Release -DAMDGPU_TARGETS=${{ matrix.gpu_target }} -DGGML_RPC=ON
+          cmake -G "Unix Makefiles" -B build -S . -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" -DGGML_HIP=ON -DCMAKE_BUILD_TYPE=Release -DAMDGPU_TARGETS=${{ matrix.gpu_target }} -DGGML_RPC=ON
           cmake --build build -j ${env:NUMBER_OF_PROCESSORS}
           md "build\bin\rocblas\library\"
           cp "${env:HIP_PATH}\bin\hipblas.dll" "build\bin\"

package/src/llama.cpp/.github/workflows/docker.yml

@@ -43,6 +43,9 @@ jobs:
           - { tag: "light-cuda", dockerfile: ".devops/llama-cli-cuda.Dockerfile", platforms: "linux/amd64" }
           - { tag: "server-cuda", dockerfile: ".devops/llama-server-cuda.Dockerfile", platforms: "linux/amd64" }
           - { tag: "full-cuda", dockerfile: ".devops/full-cuda.Dockerfile", platforms: "linux/amd64" }
+          - { tag: "light-musa", dockerfile: ".devops/llama-cli-musa.Dockerfile", platforms: "linux/amd64" }
+          - { tag: "server-musa", dockerfile: ".devops/llama-server-musa.Dockerfile", platforms: "linux/amd64" }
+          - { tag: "full-musa", dockerfile: ".devops/full-musa.Dockerfile", platforms: "linux/amd64" }
           # Note: the rocm images are failing due to a compiler error and are disabled until this is fixed to allow the workflow to complete
           #- { tag: "light-rocm", dockerfile: ".devops/llama-cli-rocm.Dockerfile", platforms: "linux/amd64,linux/arm64" }
           #- { tag: "server-rocm", dockerfile: ".devops/llama-server-rocm.Dockerfile", platforms: "linux/amd64,linux/arm64" }

package/src/llama.cpp/CMakeLists.txt

@@ -63,7 +63,7 @@ option(LLAMA_SANITIZE_ADDRESS   "llama: enable address sanitizer"   OFF)
 option(LLAMA_SANITIZE_UNDEFINED "llama: enable undefined sanitizer" OFF)
 
 # utils
-option(LLAMA_BUILD_COMMON "llama: build common utils library" ON)
+option(LLAMA_BUILD_COMMON "llama: build common utils library" ${LLAMA_STANDALONE})
 
 # extra artifacts
 option(LLAMA_BUILD_TESTS    "llama: build tests"          ${LLAMA_STANDALONE})
@@ -88,6 +88,10 @@ if (NOT DEFINED GGML_LLAMAFILE)
     set(GGML_LLAMAFILE_DEFAULT ON)
 endif()
 
+if (NOT DEFINED GGML_AMX)
+    set(GGML_AMX ON)
+endif()
+
 if (NOT DEFINED GGML_CUDA_GRAPHS)
     set(GGML_CUDA_GRAPHS_DEFAULT ON)
 endif()
@@ -136,7 +140,6 @@ set(LLAMA_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR} CACHE PATH "Location o
 set(LLAMA_LIB_INSTALL_DIR     ${CMAKE_INSTALL_LIBDIR}     CACHE PATH "Location of library files")
 set(LLAMA_BIN_INSTALL_DIR     ${CMAKE_INSTALL_BINDIR}     CACHE PATH "Location of binary  files")
 
-
 # At the moment some compile definitions are placed within the ggml/src
 # directory but not exported on the `ggml` target. This could be improved by
 # determining _precisely_ which defines are necessary for the llama-config
@@ -201,12 +204,12 @@ if (LLAMA_BUILD_COMMON)
     add_subdirectory(common)
 endif()
 
-if (LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION)
+if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION)
     include(CTest)
     add_subdirectory(tests)
 endif()
 
-if (LLAMA_BUILD_EXAMPLES)
+if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_EXAMPLES)
     add_subdirectory(examples)
     add_subdirectory(pocs)
 endif()

package/src/llama.cpp/cmake/arm64-apple-clang.cmake

@@ -0,0 +1,16 @@
+set( CMAKE_SYSTEM_NAME Darwin )
+set( CMAKE_SYSTEM_PROCESSOR arm64 )
+
+set( target arm64-apple-darwin-macho )
+
+set( CMAKE_C_COMPILER    clang )
+set( CMAKE_CXX_COMPILER  clang++ )
+
+set( CMAKE_C_COMPILER_TARGET   ${target} )
+set( CMAKE_CXX_COMPILER_TARGET ${target} )
+
+set( arch_c_flags "-march=armv8.4-a -fvectorize -ffp-model=fast -fno-finite-math-only" )
+set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function" )
+
+set( CMAKE_C_FLAGS_INIT   "${arch_c_flags} ${warn_c_flags}" )
+set( CMAKE_CXX_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" )

package/src/llama.cpp/common/CMakeLists.txt

@@ -66,8 +66,6 @@ add_library(${TARGET} STATIC
     ngram-cache.h
     sampling.cpp
     sampling.h
-    train.cpp
-    train.h
     )
 
 if (BUILD_SHARED_LIBS)