@fugood/llama.node 0.0.1-alpha.3 → 0.1.0

package/CMakeLists.txt

@@ -26,7 +26,7 @@ string(REPLACE "i686" "ia32" ARCH ${ARCH})
 string(REPLACE "i386" "ia32" ARCH ${ARCH})
 string(REPLACE "armv7l" "arm" ARCH ${ARCH})
 string(REPLACE "arm" "arm" ARCH ${ARCH})
-string(REPLACE "arm64ex" "arm64" ARCH ${ARCH})
+string(REPLACE "arm64x" "arm64" ARCH ${ARCH})
 string(REPLACE "aarch64" "arm64" ARCH ${ARCH})
 
 if(DEFINED VARIANT)
@@ -58,6 +58,12 @@ include_directories(${CMAKE_JS_INC})
 # flags: -fPIC
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
+# VULKAN_SDK
+if (VULKAN_SDK)
+  set(ENV{VULKAN_SDK} ${VULKAN_SDK})
+  find_package(Vulkan REQUIRED)
+endif()
+
 set(LLAMA_STATIC ON CACHE BOOL "Build llama as static library")
 add_subdirectory("src/llama.cpp")
 
@@ -81,31 +87,54 @@ add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES} ${CMAKE_JS_SRC})
 set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "" SUFFIX ".node")
 target_link_libraries(${PROJECT_NAME} ${CMAKE_JS_LIB} llama ggml common)
 
+add_custom_target(copy_assets ALL DEPENDS ${PROJECT_NAME})
+
+add_custom_command(
+  TARGET copy_assets
+  COMMAND ${CMAKE_COMMAND} -E remove_directory ${PLATFORM_BINARY_DIR}
+  COMMENT "Cleaning bin folder"
+)
+
 if(MSVC AND CMAKE_JS_NODELIB_DEF AND CMAKE_JS_NODELIB_TARGET)
   # Generate node.lib
   execute_process(COMMAND ${CMAKE_AR} /def:${CMAKE_JS_NODELIB_DEF} /out:${CMAKE_JS_NODELIB_TARGET} ${CMAKE_STATIC_LINKER_FLAGS})
   # copy target to bin folder
   get_filename_component(CMAKE_JS_NODELIB_TARGET_NAME ${CMAKE_JS_NODELIB_TARGET} NAME)
-  add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
+  add_custom_command(TARGET copy_assets
     COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_JS_NODELIB_TARGET} ${PLATFORM_BINARY_DIR}/${CMAKE_JS_NODELIB_TARGET_NAME}
     COMMENT "Copying to bin folder"
   )
 endif()
 
 # copy target to bin folder
-add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
+add_custom_command(TARGET copy_assets
   COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:${PROJECT_NAME}> ${PLATFORM_BINARY_DIR}/$<TARGET_FILE_NAME:${PROJECT_NAME}>
   COMMENT "Copying to bin folder"
 )
 
 if (LLAMA_METAL)
   # copy ${CMAKE_BINARY_DIR}/bin/default.metallib
-  add_custom_target(copy_metallib)
   add_custom_command(
-    TARGET copy_metallib
+    TARGET copy_assets
     COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/bin/default.metallib ${PLATFORM_BINARY_DIR}/default.metallib
     COMMENT "Copying default.metallib to bin folder"
   )
-  add_dependencies(copy_metallib ggml-metal)
-  add_dependencies(${PROJECT_NAME} copy_metallib)
+  add_dependencies(copy_assets ggml-metal)
+endif()
+
+if (LLAMA_CLBLAST)
+  find_package(CLBlast)
+  if (CLBlast_FOUND)
+    message(STATUS "CLBlast found: ${CLBlast_DIR}")
+    file(
+      GLOB CLBlast_SO_FILES
+      ${CLBlast_DIR}/../../../bin/clblast.dll
+      ${CLBlast_DIR}/../../../lib/libclblast.so
+    )
+    add_custom_command(
+      TARGET copy_assets
+      COMMAND ${CMAKE_COMMAND} -E copy ${CLBlast_SO_FILES} ${PLATFORM_BINARY_DIR}
+      COMMENT "Copying CLBlast SO files to bin folder"
+    )
+  endif()
 endif()

package/README.md

@@ -1,5 +1,9 @@
 # llama.node
 
+[![CI](https://github.com/mybigday/llama.node/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/mybigday/llama.node/actions/workflows/ci.yml)
+[![NPM Version](https://img.shields.io/npm/v/%40fugood%2Fllama.node)](https://www.npmjs.com/package/@fugood/llama.node)
+![NPM Downloads](https://img.shields.io/npm/dw/%40fugood%2Fllama.node)
+
 Node binding of [llama.cpp](https://github.com/ggerganov/llama.cpp).
 
 [llama.cpp](https://github.com/ggerganov/llama.cpp): Inference of [LLaMA](https://arxiv.org/abs/2302.13971) model in pure C/C++
@@ -22,6 +26,7 @@ const context = await loadModel({
   n_ctx: 2048,
   n_gpu_layers: 1, // > 0: enable GPU
   // embedding: true, // use embedding
+  // lib_variant: 'opencl', // Change backend
 })
 
 // Do completion
@@ -40,6 +45,10 @@ const { text, timings } = await context.completion(
 console.log('Result:', text)
 ```
 
+## Lib Variants
+
+- [x] `default`: General usage, Supported GPU: Metal (macOS) and Vulkan (Linux / Windows)
+
 ## License
 
 MIT

package/lib/binding.js

@@ -33,13 +33,30 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadModule = void 0;
+const path = __importStar(require("path"));
+const setupEnv = (variant) => {
+    var _a, _b;
+    const postfix = variant ? `-${variant}` : '';
+    const binPath = path.resolve(__dirname, `../bin/${process.platform}${postfix}/${process.arch}/`);
+    const systemPathEnv = (_b = (_a = process.env.PATH) !== null && _a !== void 0 ? _a : process.env.Path) !== null && _b !== void 0 ? _b : '';
+    if (!systemPathEnv.includes(binPath)) {
+        if (process.platform === 'win32') {
+            process.env.Path = `${binPath};${systemPathEnv}`;
+        }
+        else {
+            process.env.PATH = `${binPath}:${systemPathEnv}`;
+        }
+    }
+};
 const loadModule = (variant) => __awaiter(void 0, void 0, void 0, function* () {
     try {
-        if (variant) {
+        if (variant && variant !== 'default') {
+            setupEnv(variant);
             return yield Promise.resolve(`${`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s)));
         }
     }
     catch (_a) { } // ignore errors and try the common path
+    setupEnv();
     return yield Promise.resolve(`${`../bin/${process.platform}/${process.arch}/llama-node.node`}`).then(s => __importStar(require(s)));
 });
 exports.loadModule = loadModule;

package/lib/binding.ts

@@ -1,3 +1,5 @@
+import * as path from 'path'
+
 export type LlamaModelOptions = {
   model: string
   embedding?: boolean
@@ -21,6 +23,7 @@ export type LlamaCompletionOptions = {
   max_tokens?: number
   seed?: number
   stop?: string[]
+  grammar?: string
 }
 
 export type LlamaCompletionResult = {
@@ -48,11 +51,28 @@ export interface Module {
   LlamaContext: LlamaContext
 }
 
-export const loadModule = async (variant?: string): Promise<Module> => {
+export type LibVariant = 'default' | 'opencl'
+
+const setupEnv = (variant?: string) => {
+  const postfix = variant ? `-${variant}` : ''
+  const binPath = path.resolve(__dirname, `../bin/${process.platform}${postfix}/${process.arch}/`)
+  const systemPathEnv = process.env.PATH ?? process.env.Path ?? ''
+  if (!systemPathEnv.includes(binPath)) {
+    if (process.platform === 'win32') {
+      process.env.Path = `${binPath};${systemPathEnv}`
+    } else {
+      process.env.PATH = `${binPath}:${systemPathEnv}`
+    }
+  }
+}
+
+export const loadModule = async (variant?: LibVariant): Promise<Module> => {
   try {
-    if (variant) {
+    if (variant && variant !== 'default') {
+      setupEnv(variant)
       return await import(`../bin/${process.platform}-${variant}/${process.arch}/llama-node.node`) as Module
     }
   } catch {} // ignore errors and try the common path
+  setupEnv()
   return await import(`../bin/${process.platform}/${process.arch}/llama-node.node`) as Module
 }

package/lib/index.ts

@@ -1,10 +1,10 @@
 import { loadModule, LlamaModelOptions } from './binding'
-import type { Module, LlamaContext } from './binding'
+import type { Module, LlamaContext, LibVariant } from './binding'
 
 export * from './binding'
 
 export interface LlamaModelOptionsExtended extends LlamaModelOptions {
-  lib_variant?: string
+  lib_variant?: LibVariant
 }
 
 const mods: { [key: string]: Module } = {}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "0.0.1-alpha.3",
+  "version": "0.1.0",
   "description": "Llama.cpp for Node.js",
   "main": "lib/index.js",
   "scripts": {
@@ -39,8 +39,6 @@
   },
   "files": [
     "bin/**/*",
-    "scripts/*.js",
-    "scripts/*.ts",
     "src/**/*.{c,cc,cpp,h,hh,hpp,txt,cmake}",
     "lib/*.js",
     "lib/*.ts",
@@ -62,5 +60,19 @@
   },
   "dependencies": {
     "node-addon-api": "^8.0.0"
+  },
+  "jest": {
+    "testEnvironment": "node",
+    "moduleFileExtensions": [
+      "ts",
+      "tsx",
+      "js",
+      "jsx",
+      "json",
+      "node"
+    ],
+    "testMatch": [
+      "**/*.test.ts"
+    ]
   }
 }

package/src/LlamaCompletionWorker.cpp

@@ -58,7 +58,7 @@ void LlamaCompletionWorker::Execute() {
   const auto n_keep = _params.n_keep;
   size_t n_cur = 0;
   size_t n_input = 0;
-  const auto model = llama_get_model(_sess->context());
+  const auto model = _sess->model();
   const bool add_bos = llama_should_add_bos_token(model);
   auto ctx = _sess->context();
 
@@ -110,6 +110,7 @@ void LlamaCompletionWorker::Execute() {
     // sample the next token
     const llama_token new_token_id =
         llama_sampling_sample(sampling.get(), ctx, nullptr);
+    llama_sampling_accept(sampling.get(), ctx, new_token_id, true);
     // prepare the next batch
     embd->emplace_back(new_token_id);
     auto token = llama_token_to_piece(ctx, new_token_id);
@@ -143,6 +144,9 @@ void LlamaCompletionWorker::Execute() {
   }
   const auto t_main_end = ggml_time_us();
   _sess->get_mutex().unlock();
+  if (_onComplete) {
+    _onComplete();
+  }
 }
 
 void LlamaCompletionWorker::OnOK() {

package/src/LlamaCompletionWorker.h

@@ -1,4 +1,5 @@
 #include "common.hpp"
+#include <functional>
 
 struct CompletionResult {
   std::string text = "";
@@ -18,6 +19,8 @@ public:
 
   inline void Stop() { _stop = true; }
 
+  inline void onComplete(std::function<void()> cb) { _onComplete = cb; }
+
 protected:
   void Execute();
   void OnOK();
@@ -30,5 +33,6 @@ private:
   Napi::ThreadSafeFunction _tsfn;
   bool _has_callback = false;
   bool _stop = false;
+  std::function<void()> _onComplete;
   CompletionResult _result;
 };

package/src/LlamaContext.cpp

@@ -70,7 +70,7 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
         .ThrowAsJavaScriptException();
   }
 
-  _sess = std::make_shared<LlamaSession>(ctx, params);
+  _sess = std::make_shared<LlamaSession>(model, ctx, params);
   _info = get_system_info(params);
 }
 
@@ -93,6 +93,10 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
     Napi::TypeError::New(env, "Context is disposed")
         .ThrowAsJavaScriptException();
   }
+  if (_wip != nullptr) {
+    Napi::TypeError::New(env, "Another completion is in progress")
+        .ThrowAsJavaScriptException();
+  }
   auto options = info[0].As<Napi::Object>();
 
   gpt_params params = _sess->params();
@@ -143,6 +147,7 @@ Napi::Value LlamaContext::Completion(const Napi::CallbackInfo &info) {
       new LlamaCompletionWorker(info, _sess, callback, params, stop_words);
   worker->Queue();
   _wip = worker;
+  worker->onComplete([this]() { _wip = nullptr; });
   return worker->Promise();
 }
 
@@ -163,6 +168,12 @@ Napi::Value LlamaContext::SaveSession(const Napi::CallbackInfo &info) {
     Napi::TypeError::New(env, "Context is disposed")
         .ThrowAsJavaScriptException();
   }
+#ifdef GGML_USE_VULKAN
+  if (_sess->params().n_gpu_layers > 0) {
+    Napi::TypeError::New(env, "Vulkan cannot save session")
+        .ThrowAsJavaScriptException();
+  }
+#endif
   auto *worker = new SaveSessionWorker(info, _sess);
   worker->Queue();
   return worker->Promise();
@@ -178,6 +189,12 @@ Napi::Value LlamaContext::LoadSession(const Napi::CallbackInfo &info) {
     Napi::TypeError::New(env, "Context is disposed")
         .ThrowAsJavaScriptException();
   }
+#ifdef GGML_USE_VULKAN
+  if (_sess->params().n_gpu_layers > 0) {
+    Napi::TypeError::New(env, "Vulkan cannot load session")
+        .ThrowAsJavaScriptException();
+  }
+#endif
   auto *worker = new LoadSessionWorker(info, _sess);
   worker->Queue();
   return worker->Promise();

package/src/common.hpp

@@ -46,32 +46,36 @@ constexpr T get_option(const Napi::Object &options, const std::string &name,
 
 class LlamaSession {
 public:
-  LlamaSession(llama_context *ctx, gpt_params params)
-      : ctx_(LlamaCppContext(ctx, llama_free)), params_(params) {
+  LlamaSession(llama_model *model, llama_context *ctx, gpt_params params)
+      : model_(LlamaCppModel(model, llama_free_model)), ctx_(LlamaCppContext(ctx, llama_free)), params_(params) {
     tokens_.reserve(params.n_ctx);
   }
 
   ~LlamaSession() { dispose(); }
 
-  llama_context *context() { return ctx_.get(); }
+  inline llama_context *context() { return ctx_.get(); }
 
-  std::vector<llama_token>* tokens_ptr() { return &tokens_; }
+  inline llama_model *model() { return model_.get(); }
 
-  void set_tokens(std::vector<llama_token> tokens) {
+  inline std::vector<llama_token>* tokens_ptr() { return &tokens_; }
+
+  inline void set_tokens(std::vector<llama_token> tokens) {
     tokens_ = std::move(tokens);
   }
 
-  const gpt_params &params() const { return params_; }
+  inline const gpt_params &params() const { return params_; }
 
-  std::mutex &get_mutex() { return mutex; }
+  inline std::mutex &get_mutex() { return mutex; }
 
   void dispose() {
     std::lock_guard<std::mutex> lock(mutex);
     tokens_.clear();
     ctx_.reset();
+    model_.reset();
   }
 
 private:
+  LlamaCppModel model_;
   LlamaCppContext ctx_;
   const gpt_params params_;
   std::vector<llama_token> tokens_{};

package/src/llama.cpp/CMakeLists.txt

@@ -43,11 +43,7 @@ else()
     set(LLAMA_METAL_DEFAULT OFF)
 endif()
 
-if (CMAKE_SYSTEM_NAME MATCHES "ANDROID")
-    set(LLAMA_LLAMAFILE_DEFAULT OFF)
-else()
-    set(LLAMA_LLAMAFILE_DEFAULT ON)
-endif()
+set(LLAMA_LLAMAFILE_DEFAULT ON)
 
 # general
 option(BUILD_SHARED_LIBS                "build shared libraries"                                OFF)
@@ -107,6 +103,8 @@ set(LLAMA_CUDA_KQUANTS_ITER "2" CACHE STRING "llama: iters./thread per block for
 set(LLAMA_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING
                                              "llama: max. batch size for using peer access")
 option(LLAMA_CUDA_NO_PEER_COPY               "llama: do not use peer to peer copies"            OFF)
+option(LLAMA_CUDA_NO_VMM                     "llama: do not try to use CUDA VMM"                OFF)
+
 option(LLAMA_CURL                            "llama: use libcurl to download model from an URL" OFF)
 option(LLAMA_HIPBLAS                         "llama: use hipBLAS"                               OFF)
 option(LLAMA_HIP_UMA                         "llama: use HIP unified memory architecture"       OFF)
@@ -407,12 +405,16 @@ if (LLAMA_CUDA)
         list(APPEND GGML_SOURCES_CUDA "ggml-cuda.cu")
 
         add_compile_definitions(GGML_USE_CUDA)
+        add_compile_definitions(GGML_CUDA_USE_GRAPHS)
         if (LLAMA_CUDA_FORCE_DMMV)
             add_compile_definitions(GGML_CUDA_FORCE_DMMV)
         endif()
         if (LLAMA_CUDA_FORCE_MMQ)
             add_compile_definitions(GGML_CUDA_FORCE_MMQ)
         endif()
+        if (LLAMA_CUDA_NO_VMM)
+            add_compile_definitions(GGML_CUDA_NO_VMM)
+        endif()
         add_compile_definitions(GGML_CUDA_DMMV_X=${LLAMA_CUDA_DMMV_X})
         add_compile_definitions(GGML_CUDA_MMV_Y=${LLAMA_CUDA_MMV_Y})
         if (DEFINED LLAMA_CUDA_DMMV_Y)
@@ -429,7 +431,7 @@ if (LLAMA_CUDA)
 
         if (LLAMA_STATIC)
             if (WIN32)
-                # As of 12.3.1 CUDA Tookit for Windows does not offer a static cublas library
+                # As of 12.3.1 CUDA Toolkit for Windows does not offer a static cublas library
                 set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cudart_static CUDA::cublas CUDA::cublasLt)
             else ()
                 set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cudart_static CUDA::cublas_static CUDA::cublasLt_static)
@@ -438,7 +440,11 @@ if (LLAMA_CUDA)
             set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cudart CUDA::cublas CUDA::cublasLt)
         endif()
 
-        set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cuda_driver)
+        if (LLAMA_CUDA_NO_VMM)
+            # No VMM requested, no need to link directly with the cuda driver lib (libcuda.so)
+        else()
+            set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cuda_driver) # required by cuDeviceGetAttribute(), cuMemGetAllocationGranularity(...), ...
+        endif()
 
     if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
         # 52 == lowest CUDA 12 standard