@fugood/llama.node 1.3.3 → 1.3.5

package/CMakeLists.txt

@@ -120,16 +120,20 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND NOT DEFINED GGML_OPENMP OR GGML_O
 endif()
 
 set(LLAMA_BUILD_COMMON ON CACHE BOOL "Build common")
-
-set(LLAMA_BUILD_TOOLS ON CACHE BOOL "Build tools")
-
+set(LLAMA_BUILD_TOOLS OFF CACHE BOOL "Build tools")
+set(LLAMA_BUILD_TESTS OFF CACHE BOOL "Build tests")
+set(LLAMA_BUILD_SERVER OFF CACHE BOOL "Build server")
+set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "Build examples")
 set(LLAMA_CURL OFF CACHE BOOL "Build curl")
 
+set(LLAMA_INSTALL_VERSION "0.0.0") # TODO: Set the version number (0.0.<BUILD_NUMBER>)
+
 set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libraries")
 
 add_definitions(-DGGML_MAX_NAME=80)
 
 add_subdirectory("src/llama.cpp")
+add_subdirectory("src/llama.cpp/tools/mtmd")
 
 include_directories(
   ${CMAKE_JS_INC}

package/lib/binding.js

@@ -51,7 +51,7 @@ const getPlatformPackageName = (variant) => {
 };
 const loadPlatformPackage = (packageName) => __awaiter(void 0, void 0, void 0, function* () {
     try {
-        return yield Promise.resolve(`${packageName}`).then(s => __importStar(require(s)));
+        return (yield Promise.resolve(`${packageName}`).then(s => __importStar(require(s))));
     }
     catch (error) {
         return null;

package/lib/binding.ts

@@ -1,9 +1,9 @@
 export type MessagePart = {
-  type: string,
-  text?: string,
+  type: string
+  text?: string
   image_url?: {
     url?: string
-  },
+  }
   input_audio?: {
     format: string
     data?: string
@@ -70,6 +70,12 @@ export type LlamaModelOptions = {
    * Number of layers to keep MoE weights on CPU
    */
   n_cpu_moe?: number
+  /**
+   * List of device names to use for offloading
+   * Device names can be obtained from getBackendDevicesInfo()
+   * Example: ['Metal', 'BLAS', 'CPU']
+   */
+  devices?: string[]
   use_mlock?: boolean
   use_mmap?: boolean
   vocab_only?: boolean
@@ -375,9 +381,13 @@ export type ToolCall = {
 }
 
 export interface LlamaContext {
-  new (options: LlamaModelOptions, onProgress?: (progress: number) => void): LlamaContext
+  new (
+    options: LlamaModelOptions,
+    onProgress?: (progress: number) => void,
+  ): LlamaContext
   getSystemInfo(): string
   getModelInfo(): ModelInfo
+  getUsedDevices(): string[]
   getFormattedChat(
     messages: ChatMessage[],
     chat_template?: string,
@@ -400,8 +410,15 @@ export interface LlamaContext {
   stopCompletion(): void
   tokenize(text: string, media_paths?: string[]): Promise<TokenizeResult>
   detokenize(tokens: number[]): Promise<string>
-  embedding(text: string, params?: { embd_normalize?: number }): Promise<EmbeddingResult>
-  rerank(query: string, documents: string[], params?: RerankParams): Promise<RerankResult[]>
+  embedding(
+    text: string,
+    params?: { embd_normalize?: number },
+  ): Promise<EmbeddingResult>
+  rerank(
+    query: string,
+    documents: string[],
+    params?: RerankParams,
+  ): Promise<RerankResult[]>
   saveSession(path: string): Promise<void>
   loadSession(path: string): Promise<void>
   release(): Promise<void>
@@ -440,7 +457,7 @@ export interface LlamaContext {
    * @param options Object containing path and optional n_batch
    * @returns boolean indicating if loading was successful
    */
-  initVocoder(options: { path: string, n_batch?: number }): boolean
+  initVocoder(options: { path: string; n_batch?: number }): boolean
 
   /**
    * Unload the vocoder model
@@ -459,7 +476,10 @@ export interface LlamaContext {
    * @param text Text to complete
    * @returns Formatted audio completion
    */
-  getFormattedAudioCompletion(speaker: string|null, text: string): {
+  getFormattedAudioCompletion(
+    speaker: string | null,
+    text: string,
+  ): {
     prompt: string
     grammar?: string
   }
@@ -476,7 +496,7 @@ export interface LlamaContext {
    * @param tokens Tokens to decode
    * @returns Promise resolving to decoded audio tokens
    */
-  decodeAudioTokens(tokens: number[]|Int32Array): Promise<Float32Array>
+  decodeAudioTokens(tokens: number[] | Int32Array): Promise<Float32Array>
 
   // Parallel decoding methods
 
@@ -485,7 +505,7 @@ export interface LlamaContext {
    * @param params Configuration for parallel mode
    * @returns boolean indicating if successful
    */
-  enableParallelMode(params: { n_parallel?: number, n_batch?: number }): boolean
+  enableParallelMode(params: { n_parallel?: number; n_batch?: number }): boolean
 
   /**
    * Disable parallel decoding mode
@@ -563,9 +583,11 @@ const getPlatformPackageName = (variant?: LibVariant): string => {
   return `@fugood/node-llama-${platform}-${arch}${variantSuffix}`
 }
 
-const loadPlatformPackage = async (packageName: string): Promise<Module | null> => {
+const loadPlatformPackage = async (
+  packageName: string,
+): Promise<Module | null> => {
   try {
-    return await import(packageName) as Module
+    return (await import(packageName)) as Module
   } catch (error) {
     return null
   }
@@ -579,7 +601,9 @@ export const loadModule = async (variant?: LibVariant): Promise<Module> => {
 
   module = await loadPlatformPackage(getPlatformPackageName())
   if (module) {
-    console.warn(`Not found package for variant "${variant}", fallback to default`)
+    console.warn(
+      `Not found package for variant "${variant}", fallback to default`,
+    )
     return module
   }
 
@@ -588,7 +612,9 @@ export const loadModule = async (variant?: LibVariant): Promise<Module> => {
   return (await import('../build/Release/index.node')) as Module
 }
 
-export const isLibVariantAvailable = async (variant?: LibVariant): Promise<boolean> => {
+export const isLibVariantAvailable = async (
+  variant?: LibVariant,
+): Promise<boolean> => {
   if (variant && variant !== 'default') {
     const module = await loadPlatformPackage(getPlatformPackageName(variant))
     return module != null

package/lib/index.js

@@ -76,6 +76,9 @@ class LlamaContextWrapper {
     getModelInfo() {
         return this.ctx.getModelInfo();
     }
+    getUsedDevices() {
+        return this.ctx.getUsedDevices();
+    }
     isJinjaSupported() {
         const { minja } = this.ctx.getModelInfo().chatTemplates;
         return !!(minja === null || minja === void 0 ? void 0 : minja.toolUse) || !!(minja === null || minja === void 0 ? void 0 : minja.default);
@@ -85,7 +88,7 @@ class LlamaContextWrapper {
     }
     getFormattedChat(messages, template, params) {
         var _a;
-        const { messages: chat, has_media, media_paths, } = (0, utils_1.formatMediaChat)(messages);
+        const { messages: chat, has_media, media_paths } = (0, utils_1.formatMediaChat)(messages);
         const useJinja = this.isJinjaSupported() && (params === null || params === void 0 ? void 0 : params.jinja);
         let tmpl;
         if (template)

package/lib/index.ts

@@ -94,6 +94,10 @@ class LlamaContextWrapper {
     return this.ctx.getModelInfo()
   }
 
+  getUsedDevices(): string[] {
+    return this.ctx.getUsedDevices()
+  }
+
   isJinjaSupported(): boolean {
     const { minja } = this.ctx.getModelInfo().chatTemplates
     return !!minja?.toolUse || !!minja?.default
@@ -118,11 +122,7 @@ class LlamaContextWrapper {
       chat_template_kwargs?: Record<string, string>
     },
   ): FormattedChatResult {
-    const {
-      messages: chat,
-      has_media,
-      media_paths,
-    } = formatMediaChat(messages)
+    const { messages: chat, has_media, media_paths } = formatMediaChat(messages)
 
     const useJinja = this.isJinjaSupported() && params?.jinja
     let tmpl
@@ -169,8 +169,9 @@ class LlamaContextWrapper {
     options: LlamaCompletionOptions,
     callback?: (token: LlamaCompletionToken) => void,
   ): Promise<LlamaCompletionResult> {
-    const { messages, media_paths = options.media_paths } =
-      formatMediaChat(options.messages)
+    const { messages, media_paths = options.media_paths } = formatMediaChat(
+      options.messages,
+    )
     return this.ctx.completion(
       {
         ...options,
@@ -196,7 +197,10 @@ class LlamaContextWrapper {
     return this.ctx.detokenize(tokens)
   }
 
-  embedding(text: string, params?: { embd_normalize?: number }): Promise<EmbeddingResult> {
+  embedding(
+    text: string,
+    params?: { embd_normalize?: number },
+  ): Promise<EmbeddingResult> {
     return this.ctx.embedding(text, params)
   }
 
@@ -329,7 +333,7 @@ export const loadLlamaModelInfo = async (
 }
 
 export const getBackendDevicesInfo = async (
-  variant: LibVariant = 'default'
+  variant: LibVariant = 'default',
 ): Promise<import('./binding').BackendDeviceInfo[]> => {
   mods[variant] ??= await loadModule(variant)
   refreshNativeLogSetup()

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.3.3",
+  "version": "1.3.5",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -72,19 +72,19 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.3.3",
-    "@fugood/node-llama-linux-x64-vulkan": "1.3.3",
-    "@fugood/node-llama-linux-x64-cuda": "1.3.3",
-    "@fugood/node-llama-linux-arm64": "1.3.3",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.3.3",
-    "@fugood/node-llama-linux-arm64-cuda": "1.3.3",
-    "@fugood/node-llama-win32-x64": "1.3.3",
-    "@fugood/node-llama-win32-x64-vulkan": "1.3.3",
-    "@fugood/node-llama-win32-x64-cuda": "1.3.3",
-    "@fugood/node-llama-win32-arm64": "1.3.3",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.3.3",
-    "@fugood/node-llama-darwin-x64": "1.3.3",
-    "@fugood/node-llama-darwin-arm64": "1.3.3"
+    "@fugood/node-llama-linux-x64": "1.3.5",
+    "@fugood/node-llama-linux-x64-vulkan": "1.3.5",
+    "@fugood/node-llama-linux-x64-cuda": "1.3.5",
+    "@fugood/node-llama-linux-arm64": "1.3.5",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.3.5",
+    "@fugood/node-llama-linux-arm64-cuda": "1.3.5",
+    "@fugood/node-llama-win32-x64": "1.3.5",
+    "@fugood/node-llama-win32-x64-vulkan": "1.3.5",
+    "@fugood/node-llama-win32-x64-cuda": "1.3.5",
+    "@fugood/node-llama-win32-arm64": "1.3.5",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.3.5",
+    "@fugood/node-llama-darwin-x64": "1.3.5",
+    "@fugood/node-llama-darwin-arm64": "1.3.5"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/scripts/llama.cpp.patch

@@ -1,8 +1,8 @@
 diff --git a/src/llama.cpp/common/CMakeLists.txt b/src/llama.cpp/common/CMakeLists.txt
-index 706fa32ee..248459903 100644
+index bb168e835..cfc0e2c2e 100644
 --- a/src/llama.cpp/common/CMakeLists.txt
 +++ b/src/llama.cpp/common/CMakeLists.txt
-@@ -141,9 +141,16 @@ if (LLAMA_LLGUIDANCE)
+@@ -143,9 +143,16 @@ if (LLAMA_LLGUIDANCE)
      set(LLAMA_COMMON_EXTRA_LIBS ${LLAMA_COMMON_EXTRA_LIBS} llguidance ${LLGUIDANCE_PLATFORM_LIBS})
  endif ()
  
@@ -21,7 +21,7 @@ index 706fa32ee..248459903 100644
  
  #
 diff --git a/src/llama.cpp/common/chat.cpp b/src/llama.cpp/common/chat.cpp
-index 938872e82..6364f173f 100644
+index 6fa05a604..87dfa7a8b 100644
 --- a/src/llama.cpp/common/chat.cpp
 +++ b/src/llama.cpp/common/chat.cpp
 @@ -6,9 +6,6 @@
@@ -51,7 +51,7 @@ index 938872e82..6364f173f 100644
  struct templates_params {
      json messages;
      json tools;
-@@ -811,7 +798,7 @@ static std::string apply(
+@@ -817,7 +804,7 @@ static std::string apply(
          tmpl_inputs.extra_context.merge_patch(*additional_context);
      }
      // TODO: add flag to control date/time, if only for testing purposes.
@@ -61,7 +61,7 @@ index 938872e82..6364f173f 100644
      minja::chat_template_options tmpl_opts;
      // To avoid double BOS / EOS tokens, we're manually removing begining / trailing tokens
 diff --git a/src/llama.cpp/common/chat.h b/src/llama.cpp/common/chat.h
-index 50efb0d4e..f471a84c7 100644
+index 754c411e2..71241a6cc 100644
 --- a/src/llama.cpp/common/chat.h
 +++ b/src/llama.cpp/common/chat.h
 @@ -9,7 +9,18 @@
@@ -85,10 +85,10 @@ index 50efb0d4e..f471a84c7 100644
  struct common_chat_tool_call {
      std::string name;
 diff --git a/src/llama.cpp/common/common.cpp b/src/llama.cpp/common/common.cpp
-index 4dc95dcba..ea0ea86c0 100644
+index f3cc55247..65398844f 100644
 --- a/src/llama.cpp/common/common.cpp
 +++ b/src/llama.cpp/common/common.cpp
-@@ -1155,6 +1155,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
+@@ -1162,6 +1162,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
          mparams.n_gpu_layers = params.n_gpu_layers;
      }
  
@@ -97,10 +97,10 @@ index 4dc95dcba..ea0ea86c0 100644
      mparams.split_mode      = params.split_mode;
      mparams.tensor_split    = params.tensor_split;
 diff --git a/src/llama.cpp/common/common.h b/src/llama.cpp/common/common.h
-index f42c083fa..c573cc812 100644
+index de5b404dd..d30d252c9 100644
 --- a/src/llama.cpp/common/common.h
 +++ b/src/llama.cpp/common/common.h
-@@ -274,6 +274,7 @@ struct lr_opt {
+@@ -281,6 +281,7 @@ struct lr_opt {
  struct ggml_opt_optimizer_params common_opt_lr_pars(void * userdata);
  
  struct common_params {
@@ -109,7 +109,7 @@ index f42c083fa..c573cc812 100644
      int32_t n_ctx                 =  4096; // context size
      int32_t n_batch               =  2048; // logical batch size for prompt processing (must be >=32 to use BLAS)
 diff --git a/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt b/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
-index e52e050a8..c1000c162 100644
+index d0cab0bcb..48d532838 100644
 --- a/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
 +++ b/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
 @@ -106,7 +106,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name)

package/src/LlamaCompletionWorker.cpp

@@ -9,10 +9,10 @@ Napi::Array TokenProbsToArray(Napi::Env env, llama_context* ctx, const std::vect
   for (size_t i = 0; i < probs.size(); i++) {
     const auto &prob = probs[i];
     Napi::Object token_obj = Napi::Object::New(env);
-    
+
     std::string token_str = common_token_to_piece(ctx, prob.tok);
     token_obj.Set("content", Napi::String::New(env, token_str));
-    
+
     Napi::Array token_probs = Napi::Array::New(env);
     for (size_t j = 0; j < prob.probs.size(); j++) {
       const auto &p = prob.probs[j];
@@ -83,10 +83,10 @@ void LlamaCompletionWorker::Execute() {
     }
 
     auto completion = _rn_ctx->completion;
-    
+
     // Prepare completion context
     completion->rewind();
-    
+
     // Set up parameters
     _rn_ctx->params.prompt = _params.prompt;
     _rn_ctx->params.sampling = _params.sampling;
@@ -95,50 +95,50 @@ void LlamaCompletionWorker::Execute() {
     _rn_ctx->params.n_ctx = _params.n_ctx;
     _rn_ctx->params.n_batch = _params.n_batch;
     _rn_ctx->params.ctx_shift = _params.ctx_shift;
-    
+
     // Set prefill text
     completion->prefill_text = _prefill_text;
-    
+
     // Set up TTS guide tokens if enabled
     if (_has_vocoder && _rn_ctx->tts_wrapper != nullptr) {
       _rn_ctx->tts_wrapper->guide_tokens = _guide_tokens;
       _rn_ctx->tts_wrapper->next_token_uses_guide_token = true;
     }
-    
+
     // Initialize sampling
     if (!completion->initSampling()) {
       SetError("Failed to initialize sampling");
       return;
     }
-    
+
     // Load prompt (handles both text-only and multimodal)
     completion->loadPrompt(_media_paths);
-    
+
     // Check if context is full after loading prompt
     if (completion->context_full) {
       _result.context_full = true;
       return;
     }
-    
+
     // Begin completion with chat format and reasoning settings
     completion->beginCompletion(_chat_format, common_reasoning_format_from_name(_reasoning_format), _thinking_forced_open);
-    
+
     // Main completion loop
     int token_count = 0;
     const int max_tokens = _params.n_predict < 0 ? std::numeric_limits<int>::max() : _params.n_predict;
     while (completion->has_next_token && !_interrupted && token_count < max_tokens) {
       // Get next token using rn-llama completion
       rnllama::completion_token_output token_output = completion->doCompletion();
-      
+
       if (token_output.tok == -1) {
         break;
       }
-      
+
       token_count++;
-      
+
       std::string token_text = common_token_to_piece(_rn_ctx->ctx, token_output.tok);
       _result.text += token_text;
-      
+
       // Check for stopping strings after adding the token
       if (!_stop_words.empty()) {
         size_t stop_pos = completion->findStoppingStrings(_result.text, token_text.size(), rnllama::STOP_FULL);
@@ -148,7 +148,7 @@ void LlamaCompletionWorker::Execute() {
           break;
         }
       }
-      
+
       // Handle streaming callback
       if (_has_callback && !completion->incomplete) {
         struct TokenData {
@@ -160,9 +160,9 @@ void LlamaCompletionWorker::Execute() {
           std::vector<rnllama::completion_token_output> completion_probabilities;
           llama_context* ctx;
         };
-        
+
         auto partial_output = completion->parseChatOutput(true);
-        
+
         // Extract completion probabilities if n_probs > 0, similar to iOS implementation
         std::vector<rnllama::completion_token_output> probs_output;
         if (_rn_ctx->params.sampling.n_probs > 0) {
@@ -171,23 +171,23 @@ void LlamaCompletionWorker::Execute() {
           size_t probs_stop_pos = std::min(_sent_token_probs_index + to_send_toks.size(), completion->generated_token_probs.size());
           if (probs_pos < probs_stop_pos) {
             probs_output = std::vector<rnllama::completion_token_output>(
-              completion->generated_token_probs.begin() + probs_pos, 
+              completion->generated_token_probs.begin() + probs_pos,
               completion->generated_token_probs.begin() + probs_stop_pos
             );
           }
           _sent_token_probs_index = probs_stop_pos;
         }
-        
+
         TokenData *token_data = new TokenData{
-          token_text, 
-          partial_output.content, 
-          partial_output.reasoning_content, 
-          partial_output.tool_calls, 
+          token_text,
+          partial_output.content,
+          partial_output.reasoning_content,
+          partial_output.tool_calls,
           partial_output.accumulated_text,
           probs_output,
           _rn_ctx->ctx
         };
-        
+
         _tsfn.BlockingCall(token_data, [](Napi::Env env, Napi::Function jsCallback,
                                           TokenData *data) {
           auto obj = Napi::Object::New(env);
@@ -216,25 +216,25 @@ void LlamaCompletionWorker::Execute() {
             obj.Set("tool_calls", tool_calls);
           }
           obj.Set("accumulated_text", Napi::String::New(env, data->accumulated_text));
-          
+
           // Add completion_probabilities if available
           if (!data->completion_probabilities.empty()) {
             obj.Set("completion_probabilities", TokenProbsToArray(env, data->ctx, data->completion_probabilities));
           }
-          
+
           delete data;
           jsCallback.Call({obj});
         });
       }
     }
-    
+
     // Check stopping conditions
     if (token_count >= max_tokens) {
       _result.stopped_limited = true;
     } else if (!completion->has_next_token && completion->n_remain == 0) {
       _result.stopped_limited = true;
     }
-    
+
     // Set completion results from rn-llama completion context
     // tokens_evaluated should include both prompt tokens and generated tokens that were processed
     _result.tokens_evaluated = completion->num_prompt_tokens + completion->num_tokens_predicted;
@@ -245,20 +245,20 @@ void LlamaCompletionWorker::Execute() {
     _result.stopped_words = completion->stopped_word;
     _result.stopping_word = completion->stopping_word;
     _result.stopped_limited = completion->stopped_limit;
-    
+
     // Get audio tokens if TTS is enabled
     if (_has_vocoder && _rn_ctx->tts_wrapper != nullptr) {
       _result.audio_tokens = _rn_ctx->tts_wrapper->audio_tokens;
     }
-    
+    common_perf_print(_rn_ctx->ctx, _rn_ctx->completion->ctx_sampling);
     // End completion
     completion->endCompletion();
-    
+
   } catch (const std::exception &e) {
     SetError(e.what());
     return;
   }
-  
+
   if (_onComplete) {
     _onComplete();
   }