@fugood/llama.node 1.3.4 → 1.3.5

package/lib/binding.js

@@ -51,7 +51,7 @@ const getPlatformPackageName = (variant) => {
 };
 const loadPlatformPackage = (packageName) => __awaiter(void 0, void 0, void 0, function* () {
     try {
-        return yield Promise.resolve(`${packageName}`).then(s => __importStar(require(s)));
+        return (yield Promise.resolve(`${packageName}`).then(s => __importStar(require(s))));
     }
     catch (error) {
         return null;

package/lib/binding.ts

@@ -1,9 +1,9 @@
 export type MessagePart = {
-  type: string,
-  text?: string,
+  type: string
+  text?: string
   image_url?: {
     url?: string
-  },
+  }
   input_audio?: {
     format: string
     data?: string
@@ -70,6 +70,12 @@ export type LlamaModelOptions = {
    * Number of layers to keep MoE weights on CPU
    */
   n_cpu_moe?: number
+  /**
+   * List of device names to use for offloading
+   * Device names can be obtained from getBackendDevicesInfo()
+   * Example: ['Metal', 'BLAS', 'CPU']
+   */
+  devices?: string[]
   use_mlock?: boolean
   use_mmap?: boolean
   vocab_only?: boolean
@@ -375,9 +381,13 @@ export type ToolCall = {
 }
 
 export interface LlamaContext {
-  new (options: LlamaModelOptions, onProgress?: (progress: number) => void): LlamaContext
+  new (
+    options: LlamaModelOptions,
+    onProgress?: (progress: number) => void,
+  ): LlamaContext
   getSystemInfo(): string
   getModelInfo(): ModelInfo
+  getUsedDevices(): string[]
   getFormattedChat(
     messages: ChatMessage[],
     chat_template?: string,
@@ -400,8 +410,15 @@ export interface LlamaContext {
   stopCompletion(): void
   tokenize(text: string, media_paths?: string[]): Promise<TokenizeResult>
   detokenize(tokens: number[]): Promise<string>
-  embedding(text: string, params?: { embd_normalize?: number }): Promise<EmbeddingResult>
-  rerank(query: string, documents: string[], params?: RerankParams): Promise<RerankResult[]>
+  embedding(
+    text: string,
+    params?: { embd_normalize?: number },
+  ): Promise<EmbeddingResult>
+  rerank(
+    query: string,
+    documents: string[],
+    params?: RerankParams,
+  ): Promise<RerankResult[]>
   saveSession(path: string): Promise<void>
   loadSession(path: string): Promise<void>
   release(): Promise<void>
@@ -440,7 +457,7 @@ export interface LlamaContext {
    * @param options Object containing path and optional n_batch
    * @returns boolean indicating if loading was successful
    */
-  initVocoder(options: { path: string, n_batch?: number }): boolean
+  initVocoder(options: { path: string; n_batch?: number }): boolean
 
   /**
    * Unload the vocoder model
@@ -459,7 +476,10 @@ export interface LlamaContext {
    * @param text Text to complete
    * @returns Formatted audio completion
    */
-  getFormattedAudioCompletion(speaker: string|null, text: string): {
+  getFormattedAudioCompletion(
+    speaker: string | null,
+    text: string,
+  ): {
     prompt: string
     grammar?: string
   }
@@ -476,7 +496,7 @@ export interface LlamaContext {
    * @param tokens Tokens to decode
    * @returns Promise resolving to decoded audio tokens
    */
-  decodeAudioTokens(tokens: number[]|Int32Array): Promise<Float32Array>
+  decodeAudioTokens(tokens: number[] | Int32Array): Promise<Float32Array>
 
   // Parallel decoding methods
 
@@ -485,7 +505,7 @@ export interface LlamaContext {
    * @param params Configuration for parallel mode
    * @returns boolean indicating if successful
    */
-  enableParallelMode(params: { n_parallel?: number, n_batch?: number }): boolean
+  enableParallelMode(params: { n_parallel?: number; n_batch?: number }): boolean
 
   /**
    * Disable parallel decoding mode
@@ -563,9 +583,11 @@ const getPlatformPackageName = (variant?: LibVariant): string => {
   return `@fugood/node-llama-${platform}-${arch}${variantSuffix}`
 }
 
-const loadPlatformPackage = async (packageName: string): Promise<Module | null> => {
+const loadPlatformPackage = async (
+  packageName: string,
+): Promise<Module | null> => {
   try {
-    return await import(packageName) as Module
+    return (await import(packageName)) as Module
   } catch (error) {
     return null
   }
@@ -579,7 +601,9 @@ export const loadModule = async (variant?: LibVariant): Promise<Module> => {
 
   module = await loadPlatformPackage(getPlatformPackageName())
   if (module) {
-    console.warn(`Not found package for variant "${variant}", fallback to default`)
+    console.warn(
+      `Not found package for variant "${variant}", fallback to default`,
+    )
     return module
   }
 
@@ -588,7 +612,9 @@ export const loadModule = async (variant?: LibVariant): Promise<Module> => {
   return (await import('../build/Release/index.node')) as Module
 }
 
-export const isLibVariantAvailable = async (variant?: LibVariant): Promise<boolean> => {
+export const isLibVariantAvailable = async (
+  variant?: LibVariant,
+): Promise<boolean> => {
   if (variant && variant !== 'default') {
     const module = await loadPlatformPackage(getPlatformPackageName(variant))
     return module != null

package/lib/index.js

@@ -76,6 +76,9 @@ class LlamaContextWrapper {
     getModelInfo() {
         return this.ctx.getModelInfo();
     }
+    getUsedDevices() {
+        return this.ctx.getUsedDevices();
+    }
     isJinjaSupported() {
         const { minja } = this.ctx.getModelInfo().chatTemplates;
         return !!(minja === null || minja === void 0 ? void 0 : minja.toolUse) || !!(minja === null || minja === void 0 ? void 0 : minja.default);
@@ -85,7 +88,7 @@ class LlamaContextWrapper {
     }
     getFormattedChat(messages, template, params) {
         var _a;
-        const { messages: chat, has_media, media_paths, } = (0, utils_1.formatMediaChat)(messages);
+        const { messages: chat, has_media, media_paths } = (0, utils_1.formatMediaChat)(messages);
         const useJinja = this.isJinjaSupported() && (params === null || params === void 0 ? void 0 : params.jinja);
         let tmpl;
         if (template)

package/lib/index.ts

@@ -94,6 +94,10 @@ class LlamaContextWrapper {
     return this.ctx.getModelInfo()
   }
 
+  getUsedDevices(): string[] {
+    return this.ctx.getUsedDevices()
+  }
+
   isJinjaSupported(): boolean {
     const { minja } = this.ctx.getModelInfo().chatTemplates
     return !!minja?.toolUse || !!minja?.default
@@ -118,11 +122,7 @@ class LlamaContextWrapper {
       chat_template_kwargs?: Record<string, string>
     },
   ): FormattedChatResult {
-    const {
-      messages: chat,
-      has_media,
-      media_paths,
-    } = formatMediaChat(messages)
+    const { messages: chat, has_media, media_paths } = formatMediaChat(messages)
 
     const useJinja = this.isJinjaSupported() && params?.jinja
     let tmpl
@@ -169,8 +169,9 @@ class LlamaContextWrapper {
     options: LlamaCompletionOptions,
     callback?: (token: LlamaCompletionToken) => void,
   ): Promise<LlamaCompletionResult> {
-    const { messages, media_paths = options.media_paths } =
-      formatMediaChat(options.messages)
+    const { messages, media_paths = options.media_paths } = formatMediaChat(
+      options.messages,
+    )
     return this.ctx.completion(
       {
         ...options,
@@ -196,7 +197,10 @@ class LlamaContextWrapper {
     return this.ctx.detokenize(tokens)
   }
 
-  embedding(text: string, params?: { embd_normalize?: number }): Promise<EmbeddingResult> {
+  embedding(
+    text: string,
+    params?: { embd_normalize?: number },
+  ): Promise<EmbeddingResult> {
     return this.ctx.embedding(text, params)
   }
 
@@ -329,7 +333,7 @@ export const loadLlamaModelInfo = async (
 }
 
 export const getBackendDevicesInfo = async (
-  variant: LibVariant = 'default'
+  variant: LibVariant = 'default',
 ): Promise<import('./binding').BackendDeviceInfo[]> => {
   mods[variant] ??= await loadModule(variant)
   refreshNativeLogSetup()

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.3.4",
+  "version": "1.3.5",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -72,19 +72,19 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.3.4",
-    "@fugood/node-llama-linux-x64-vulkan": "1.3.4",
-    "@fugood/node-llama-linux-x64-cuda": "1.3.4",
-    "@fugood/node-llama-linux-arm64": "1.3.4",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.3.4",
-    "@fugood/node-llama-linux-arm64-cuda": "1.3.4",
-    "@fugood/node-llama-win32-x64": "1.3.4",
-    "@fugood/node-llama-win32-x64-vulkan": "1.3.4",
-    "@fugood/node-llama-win32-x64-cuda": "1.3.4",
-    "@fugood/node-llama-win32-arm64": "1.3.4",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.3.4",
-    "@fugood/node-llama-darwin-x64": "1.3.4",
-    "@fugood/node-llama-darwin-arm64": "1.3.4"
+    "@fugood/node-llama-linux-x64": "1.3.5",
+    "@fugood/node-llama-linux-x64-vulkan": "1.3.5",
+    "@fugood/node-llama-linux-x64-cuda": "1.3.5",
+    "@fugood/node-llama-linux-arm64": "1.3.5",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.3.5",
+    "@fugood/node-llama-linux-arm64-cuda": "1.3.5",
+    "@fugood/node-llama-win32-x64": "1.3.5",
+    "@fugood/node-llama-win32-x64-vulkan": "1.3.5",
+    "@fugood/node-llama-win32-x64-cuda": "1.3.5",
+    "@fugood/node-llama-win32-arm64": "1.3.5",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.3.5",
+    "@fugood/node-llama-darwin-x64": "1.3.5",
+    "@fugood/node-llama-darwin-arm64": "1.3.5"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/scripts/llama.cpp.patch

@@ -1,8 +1,8 @@
 diff --git a/src/llama.cpp/common/CMakeLists.txt b/src/llama.cpp/common/CMakeLists.txt
-index 706fa32ee..248459903 100644
+index bb168e835..cfc0e2c2e 100644
 --- a/src/llama.cpp/common/CMakeLists.txt
 +++ b/src/llama.cpp/common/CMakeLists.txt
-@@ -141,9 +141,16 @@ if (LLAMA_LLGUIDANCE)
+@@ -143,9 +143,16 @@ if (LLAMA_LLGUIDANCE)
      set(LLAMA_COMMON_EXTRA_LIBS ${LLAMA_COMMON_EXTRA_LIBS} llguidance ${LLGUIDANCE_PLATFORM_LIBS})
  endif ()
  
@@ -21,7 +21,7 @@ index 706fa32ee..248459903 100644
  
  #
 diff --git a/src/llama.cpp/common/chat.cpp b/src/llama.cpp/common/chat.cpp
-index 938872e82..6364f173f 100644
+index 6fa05a604..87dfa7a8b 100644
 --- a/src/llama.cpp/common/chat.cpp
 +++ b/src/llama.cpp/common/chat.cpp
 @@ -6,9 +6,6 @@
@@ -51,7 +51,7 @@ index 938872e82..6364f173f 100644
  struct templates_params {
      json messages;
      json tools;
-@@ -811,7 +798,7 @@ static std::string apply(
+@@ -817,7 +804,7 @@ static std::string apply(
          tmpl_inputs.extra_context.merge_patch(*additional_context);
      }
      // TODO: add flag to control date/time, if only for testing purposes.
@@ -61,7 +61,7 @@ index 938872e82..6364f173f 100644
      minja::chat_template_options tmpl_opts;
      // To avoid double BOS / EOS tokens, we're manually removing begining / trailing tokens
 diff --git a/src/llama.cpp/common/chat.h b/src/llama.cpp/common/chat.h
-index 50efb0d4e..f471a84c7 100644
+index 754c411e2..71241a6cc 100644
 --- a/src/llama.cpp/common/chat.h
 +++ b/src/llama.cpp/common/chat.h
 @@ -9,7 +9,18 @@
@@ -85,10 +85,10 @@ index 50efb0d4e..f471a84c7 100644
  struct common_chat_tool_call {
      std::string name;
 diff --git a/src/llama.cpp/common/common.cpp b/src/llama.cpp/common/common.cpp
-index 4dc95dcba..ea0ea86c0 100644
+index f3cc55247..65398844f 100644
 --- a/src/llama.cpp/common/common.cpp
 +++ b/src/llama.cpp/common/common.cpp
-@@ -1155,6 +1155,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
+@@ -1162,6 +1162,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
          mparams.n_gpu_layers = params.n_gpu_layers;
      }
  
@@ -97,10 +97,10 @@ index 4dc95dcba..ea0ea86c0 100644
      mparams.split_mode      = params.split_mode;
      mparams.tensor_split    = params.tensor_split;
 diff --git a/src/llama.cpp/common/common.h b/src/llama.cpp/common/common.h
-index f42c083fa..c573cc812 100644
+index de5b404dd..d30d252c9 100644
 --- a/src/llama.cpp/common/common.h
 +++ b/src/llama.cpp/common/common.h
-@@ -274,6 +274,7 @@ struct lr_opt {
+@@ -281,6 +281,7 @@ struct lr_opt {
  struct ggml_opt_optimizer_params common_opt_lr_pars(void * userdata);
  
  struct common_params {
@@ -109,7 +109,7 @@ index f42c083fa..c573cc812 100644
      int32_t n_ctx                 =  4096; // context size
      int32_t n_batch               =  2048; // logical batch size for prompt processing (must be >=32 to use BLAS)
 diff --git a/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt b/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
-index e52e050a8..c1000c162 100644
+index d0cab0bcb..48d532838 100644
 --- a/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
 +++ b/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
 @@ -106,7 +106,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name)

package/src/LlamaContext.cpp

@@ -105,6 +105,9 @@ void LlamaContext::Init(Napi::Env env, Napi::Object &exports) {
        InstanceMethod<&LlamaContext::GetModelInfo>(
            "getModelInfo",
            static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::GetUsedDevices>(
+           "getUsedDevices",
+           static_cast<napi_property_attributes>(napi_enumerable)),
        InstanceMethod<&LlamaContext::GetFormattedChat>(
            "getFormattedChat",
            static_cast<napi_property_attributes>(napi_enumerable)),
@@ -306,6 +309,19 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
   llama_backend_init();
   llama_numa_init(params.numa);
 
+  // Parse devices array
+  if (options.Has("devices") && options.Get("devices").IsArray()) {
+    auto devices_array = options.Get("devices").As<Napi::Array>();
+    for (size_t i = 0; i < devices_array.Length(); i++) {
+      auto device_name = devices_array.Get(i).ToString().Utf8Value();
+      auto * dev = ggml_backend_dev_by_name(device_name.c_str());
+      if (dev) {
+        params.devices.push_back(dev);
+      }
+      // Skip invalid device names silently
+    }
+  }
+
   std::vector<common_adapter_lora_info> lora;
   auto lora_path = get_option<std::string>(options, "lora", "");
   auto lora_scaled = get_option<float>(options, "lora_scaled", 1.0f);
@@ -378,6 +394,17 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
   }
   _rn_ctx->attachThreadpoolsIfAvailable();
 
+  // Collect used devices from the loaded model
+  if (_rn_ctx->llama_init.model) {
+    const auto &model_devices = _rn_ctx->llama_init.model->devices;
+    for (auto dev : model_devices) {
+      const char *dev_name = ggml_backend_dev_name(dev);
+      if (dev_name != nullptr) {
+        _used_devices.push_back(std::string(dev_name));
+      }
+    }
+  }
+
   // Release progress callback after model is loaded
   if (has_progress_callback) {
     _progress_tsfn.Release();
@@ -583,6 +610,15 @@ Napi::Value LlamaContext::GetModelInfo(const Napi::CallbackInfo &info) {
   return details;
 }
 
+// getUsedDevices(): string[]
+Napi::Value LlamaContext::GetUsedDevices(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  Napi::Array devices = Napi::Array::New(env, _used_devices.size());
+  for (size_t i = 0; i < _used_devices.size(); i++) {
+    devices[i] = Napi::String::New(env, _used_devices[i]);
+  }
+  return devices;
+}
 
 
 // getFormattedChat(

package/src/LlamaContext.h

@@ -31,6 +31,7 @@ public:
 private:
   Napi::Value GetSystemInfo(const Napi::CallbackInfo &info);
   Napi::Value GetModelInfo(const Napi::CallbackInfo &info);
+  Napi::Value GetUsedDevices(const Napi::CallbackInfo &info);
   Napi::Value GetFormattedChat(const Napi::CallbackInfo &info);
   Napi::Value Completion(const Napi::CallbackInfo &info);
   void StopCompletion(const Napi::CallbackInfo &info);
@@ -69,6 +70,7 @@ private:
   void CancelRequest(const Napi::CallbackInfo &info);
 
   std::string _info;
+  std::vector<std::string> _used_devices;
   Napi::Object _meta;
   LlamaCompletionWorker *_wip = nullptr;
 

package/src/llama.cpp/common/CMakeLists.txt

@@ -50,6 +50,8 @@ add_library(${TARGET} STATIC
     base64.hpp
     chat-parser.cpp
     chat-parser.h
+    chat-parser-xml-toolcall.h
+    chat-parser-xml-toolcall.cpp
     chat.cpp
     chat.h
     common.cpp