@fugood/llama.node 1.3.4 → 1.3.6

package/CMakeLists.txt

@@ -44,7 +44,8 @@ else()
 endif()
 
 if (TO_PACKAGE)
-  set(PLATFORM_BINARY_DIR ${CMAKE_SOURCE_DIR}/packages/node-llama-${PLATFORM}-${ARCH}${VARIANT})
+  set(PACKAGE_NAME "node-llama-${PLATFORM}-${ARCH}${VARIANT}")
+  set(PLATFORM_BINARY_DIR ${CMAKE_SOURCE_DIR}/packages/${PACKAGE_NAME})
 else()
   set(PLATFORM_BINARY_DIR ${CMAKE_SOURCE_DIR}/build/Release)
 endif()
@@ -188,6 +189,13 @@ if (NOT MSVC AND CMAKE_SYSTEM_NAME STREQUAL "Windows")
   set(CMAKE_JS_LIB win_dynamic_load)
 endif()
 
+if (TO_PACKAGE AND GGML_HEXAGON)
+  set(NODE_RPATH "node_modules/@fugood/${PACKAGE_NAME}")
+  set(ELECTRON_ASAR_RPATH "resources/app.asar.unpacked/node_modules/@fugood/${PACKAGE_NAME}")
+  set(ELECTRON_RES_RPATH "resources/node_modules/@fugood/${PACKAGE_NAME}")
+  set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-rpath,${NODE_RPATH} -Wl,-rpath,${ELECTRON_ASAR_RPATH} -Wl,-rpath,${ELECTRON_RES_RPATH}")
+endif()
+
 add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES} ${CMAKE_JS_SRC})
 set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "" SUFFIX ".node")
 target_link_libraries(${PROJECT_NAME} ${CMAKE_JS_LIB} llama ggml common mtmd ${CMAKE_THREAD_LIBS_INIT})
@@ -237,3 +245,15 @@ if (GGML_CLBLAST AND TO_PACKAGE)
     )
   endif()
 endif()
+
+if (GGML_HEXAGON)
+  get_target_property(HTP_LIBS_DIR ggml-hexagon BINARY_DIR)
+  add_custom_command(
+    TARGET copy_assets
+    COMMAND ${CMAKE_COMMAND} -E copy ${HTP_LIBS_DIR}/libggml-htp-v73.so ${PLATFORM_BINARY_DIR}
+    COMMAND ${CMAKE_COMMAND} -E copy ${HTP_LIBS_DIR}/libggml-htp-v75.so ${PLATFORM_BINARY_DIR}
+    COMMAND ${CMAKE_COMMAND} -E copy ${HTP_LIBS_DIR}/libggml-htp-v79.so ${PLATFORM_BINARY_DIR}
+    COMMAND ${CMAKE_COMMAND} -E copy ${HTP_LIBS_DIR}/libggml-htp-v81.so ${PLATFORM_BINARY_DIR}
+    COMMENT "Copying HTP libraries to bin folder"
+  )
+endif()

package/lib/binding.js

@@ -51,7 +51,7 @@ const getPlatformPackageName = (variant) => {
 };
 const loadPlatformPackage = (packageName) => __awaiter(void 0, void 0, void 0, function* () {
     try {
-        return yield Promise.resolve(`${packageName}`).then(s => __importStar(require(s)));
+        return (yield Promise.resolve(`${packageName}`).then(s => __importStar(require(s))));
     }
     catch (error) {
         return null;

package/lib/binding.ts

@@ -1,9 +1,9 @@
 export type MessagePart = {
-  type: string,
-  text?: string,
+  type: string
+  text?: string
   image_url?: {
     url?: string
-  },
+  }
   input_audio?: {
     format: string
     data?: string
@@ -25,6 +25,12 @@ export type LlamaModelOptions = {
   n_ctx?: number
   n_batch?: number
   n_ubatch?: number
+  /**
+   * CPU affinity mask
+   * Example: '0xfc'
+   */
+  cpu_mask?: string
+  cpu_strict?: boolean
   /**
    * Number of parallel sequences to support (sets n_seq_max).
    * This determines the maximum number of parallel slots that can be used.
@@ -70,6 +76,12 @@ export type LlamaModelOptions = {
    * Number of layers to keep MoE weights on CPU
    */
   n_cpu_moe?: number
+  /**
+   * List of device names to use for offloading
+   * Device names can be obtained from getBackendDevicesInfo()
+   * Example: ['Metal', 'BLAS', 'CPU']
+   */
+  devices?: string[]
   use_mlock?: boolean
   use_mmap?: boolean
   vocab_only?: boolean
@@ -375,9 +387,13 @@ export type ToolCall = {
 }
 
 export interface LlamaContext {
-  new (options: LlamaModelOptions, onProgress?: (progress: number) => void): LlamaContext
+  new (
+    options: LlamaModelOptions,
+    onProgress?: (progress: number) => void,
+  ): LlamaContext
   getSystemInfo(): string
   getModelInfo(): ModelInfo
+  getUsedDevices(): string[]
   getFormattedChat(
     messages: ChatMessage[],
     chat_template?: string,
@@ -400,8 +416,15 @@ export interface LlamaContext {
   stopCompletion(): void
   tokenize(text: string, media_paths?: string[]): Promise<TokenizeResult>
   detokenize(tokens: number[]): Promise<string>
-  embedding(text: string, params?: { embd_normalize?: number }): Promise<EmbeddingResult>
-  rerank(query: string, documents: string[], params?: RerankParams): Promise<RerankResult[]>
+  embedding(
+    text: string,
+    params?: { embd_normalize?: number },
+  ): Promise<EmbeddingResult>
+  rerank(
+    query: string,
+    documents: string[],
+    params?: RerankParams,
+  ): Promise<RerankResult[]>
   saveSession(path: string): Promise<void>
   loadSession(path: string): Promise<void>
   release(): Promise<void>
@@ -440,7 +463,7 @@ export interface LlamaContext {
    * @param options Object containing path and optional n_batch
    * @returns boolean indicating if loading was successful
    */
-  initVocoder(options: { path: string, n_batch?: number }): boolean
+  initVocoder(options: { path: string; n_batch?: number }): boolean
 
   /**
    * Unload the vocoder model
@@ -459,7 +482,10 @@ export interface LlamaContext {
    * @param text Text to complete
    * @returns Formatted audio completion
    */
-  getFormattedAudioCompletion(speaker: string|null, text: string): {
+  getFormattedAudioCompletion(
+    speaker: string | null,
+    text: string,
+  ): {
     prompt: string
     grammar?: string
   }
@@ -476,7 +502,7 @@ export interface LlamaContext {
    * @param tokens Tokens to decode
    * @returns Promise resolving to decoded audio tokens
    */
-  decodeAudioTokens(tokens: number[]|Int32Array): Promise<Float32Array>
+  decodeAudioTokens(tokens: number[] | Int32Array): Promise<Float32Array>
 
   // Parallel decoding methods
 
@@ -485,7 +511,7 @@ export interface LlamaContext {
    * @param params Configuration for parallel mode
    * @returns boolean indicating if successful
    */
-  enableParallelMode(params: { n_parallel?: number, n_batch?: number }): boolean
+  enableParallelMode(params: { n_parallel?: number; n_batch?: number }): boolean
 
   /**
    * Disable parallel decoding mode
@@ -554,7 +580,7 @@ export interface Module {
   LlamaContext: LlamaContext
 }
 
-export type LibVariant = 'default' | 'vulkan' | 'cuda'
+export type LibVariant = 'default' | 'vulkan' | 'cuda' | 'snapdragon'
 
 const getPlatformPackageName = (variant?: LibVariant): string => {
   const platform = process.platform
@@ -563,9 +589,11 @@ const getPlatformPackageName = (variant?: LibVariant): string => {
   return `@fugood/node-llama-${platform}-${arch}${variantSuffix}`
 }
 
-const loadPlatformPackage = async (packageName: string): Promise<Module | null> => {
+const loadPlatformPackage = async (
+  packageName: string,
+): Promise<Module | null> => {
   try {
-    return await import(packageName) as Module
+    return (await import(packageName)) as Module
   } catch (error) {
     return null
   }
@@ -579,7 +607,9 @@ export const loadModule = async (variant?: LibVariant): Promise<Module> => {
 
   module = await loadPlatformPackage(getPlatformPackageName())
   if (module) {
-    console.warn(`Not found package for variant "${variant}", fallback to default`)
+    console.warn(
+      `Not found package for variant "${variant}", fallback to default`,
+    )
     return module
   }
 
@@ -588,7 +618,9 @@ export const loadModule = async (variant?: LibVariant): Promise<Module> => {
   return (await import('../build/Release/index.node')) as Module
 }
 
-export const isLibVariantAvailable = async (variant?: LibVariant): Promise<boolean> => {
+export const isLibVariantAvailable = async (
+  variant?: LibVariant,
+): Promise<boolean> => {
   if (variant && variant !== 'default') {
     const module = await loadPlatformPackage(getPlatformPackageName(variant))
     return module != null

package/lib/index.js

@@ -76,6 +76,9 @@ class LlamaContextWrapper {
     getModelInfo() {
         return this.ctx.getModelInfo();
     }
+    getUsedDevices() {
+        return this.ctx.getUsedDevices();
+    }
     isJinjaSupported() {
         const { minja } = this.ctx.getModelInfo().chatTemplates;
         return !!(minja === null || minja === void 0 ? void 0 : minja.toolUse) || !!(minja === null || minja === void 0 ? void 0 : minja.default);
@@ -85,7 +88,7 @@ class LlamaContextWrapper {
     }
     getFormattedChat(messages, template, params) {
         var _a;
-        const { messages: chat, has_media, media_paths, } = (0, utils_1.formatMediaChat)(messages);
+        const { messages: chat, has_media, media_paths } = (0, utils_1.formatMediaChat)(messages);
         const useJinja = this.isJinjaSupported() && (params === null || params === void 0 ? void 0 : params.jinja);
         let tmpl;
         if (template)
@@ -198,7 +201,28 @@ const loadModel = (options, onProgress) => __awaiter(void 0, void 0, void 0, fun
     const variant = (_a = options.lib_variant) !== null && _a !== void 0 ? _a : 'default';
     (_b = mods[variant]) !== null && _b !== void 0 ? _b : (mods[variant] = yield (0, binding_1.loadModule)(options.lib_variant));
     refreshNativeLogSetup();
-    const nativeCtx = new mods[variant].LlamaContext(options, onProgress);
+    const { devices } = options;
+    let filteredDevs = [];
+    if (Array.isArray(devices)) {
+        filteredDevs = [...devices];
+        // Handle HTP* to use all HTP devices on Hexagon
+        if (variant === 'snapdragon' && devices.includes('HTP*')) {
+            const backendDevices = yield (0, exports.getBackendDevicesInfo)(variant);
+            const htpDevices = backendDevices
+                .filter((d) => d.deviceName.startsWith('HTP'))
+                .map((d) => d.deviceName);
+            filteredDevs = filteredDevs.reduce((acc, dev) => {
+                if (dev.startsWith('HTP*')) {
+                    acc.push(...htpDevices);
+                }
+                else if (!dev.startsWith('HTP')) {
+                    acc.push(dev);
+                }
+                return acc;
+            }, []);
+        }
+    }
+    const nativeCtx = new mods[variant].LlamaContext(Object.assign(Object.assign({}, options), { devices: filteredDevs.length > 0 ? filteredDevs : undefined }), onProgress);
     return new LlamaContextWrapper(nativeCtx);
 });
 exports.loadModel = loadModel;

package/lib/index.ts

@@ -94,6 +94,10 @@ class LlamaContextWrapper {
     return this.ctx.getModelInfo()
   }
 
+  getUsedDevices(): string[] {
+    return this.ctx.getUsedDevices()
+  }
+
   isJinjaSupported(): boolean {
     const { minja } = this.ctx.getModelInfo().chatTemplates
     return !!minja?.toolUse || !!minja?.default
@@ -118,11 +122,7 @@ class LlamaContextWrapper {
       chat_template_kwargs?: Record<string, string>
     },
   ): FormattedChatResult {
-    const {
-      messages: chat,
-      has_media,
-      media_paths,
-    } = formatMediaChat(messages)
+    const { messages: chat, has_media, media_paths } = formatMediaChat(messages)
 
     const useJinja = this.isJinjaSupported() && params?.jinja
     let tmpl
@@ -169,8 +169,9 @@ class LlamaContextWrapper {
     options: LlamaCompletionOptions,
     callback?: (token: LlamaCompletionToken) => void,
   ): Promise<LlamaCompletionResult> {
-    const { messages, media_paths = options.media_paths } =
-      formatMediaChat(options.messages)
+    const { messages, media_paths = options.media_paths } = formatMediaChat(
+      options.messages,
+    )
     return this.ctx.completion(
       {
         ...options,
@@ -196,7 +197,10 @@ class LlamaContextWrapper {
     return this.ctx.detokenize(tokens)
   }
 
-  embedding(text: string, params?: { embd_normalize?: number }): Promise<EmbeddingResult> {
+  embedding(
+    text: string,
+    params?: { embd_normalize?: number },
+  ): Promise<EmbeddingResult> {
     return this.ctx.embedding(text, params)
   }
 
@@ -305,7 +309,35 @@ export const loadModel = async (
   mods[variant] ??= await loadModule(options.lib_variant)
   refreshNativeLogSetup()
 
-  const nativeCtx = new mods[variant].LlamaContext(options, onProgress)
+  const { devices } = options
+  let filteredDevs: Array<string> = []
+  if (Array.isArray(devices)) {
+    filteredDevs = [...devices]
+
+    // Handle HTP* to use all HTP devices on Hexagon
+    if (variant === 'snapdragon' && devices.includes('HTP*')) {
+      const backendDevices = await getBackendDevicesInfo(variant)
+      const htpDevices = backendDevices
+        .filter((d) => d.deviceName.startsWith('HTP'))
+        .map((d) => d.deviceName)
+      filteredDevs = filteredDevs.reduce((acc, dev) => {
+        if (dev.startsWith('HTP*')) {
+          acc.push(...htpDevices)
+        } else if (!dev.startsWith('HTP')) {
+          acc.push(dev)
+        }
+        return acc
+      }, [] as Array<string>)
+    }
+  }
+
+  const nativeCtx = new mods[variant].LlamaContext(
+    {
+      ...options,
+      devices: filteredDevs.length > 0 ? filteredDevs : undefined,
+    },
+    onProgress,
+  )
   return new LlamaContextWrapper(nativeCtx)
 }
 
@@ -329,7 +361,7 @@ export const loadLlamaModelInfo = async (
 }
 
 export const getBackendDevicesInfo = async (
-  variant: LibVariant = 'default'
+  variant: LibVariant = 'default',
 ): Promise<import('./binding').BackendDeviceInfo[]> => {
   mods[variant] ??= await loadModule(variant)
   refreshNativeLogSetup()

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.3.4",
+  "version": "1.3.6",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -72,19 +72,20 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.3.4",
-    "@fugood/node-llama-linux-x64-vulkan": "1.3.4",
-    "@fugood/node-llama-linux-x64-cuda": "1.3.4",
-    "@fugood/node-llama-linux-arm64": "1.3.4",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.3.4",
-    "@fugood/node-llama-linux-arm64-cuda": "1.3.4",
-    "@fugood/node-llama-win32-x64": "1.3.4",
-    "@fugood/node-llama-win32-x64-vulkan": "1.3.4",
-    "@fugood/node-llama-win32-x64-cuda": "1.3.4",
-    "@fugood/node-llama-win32-arm64": "1.3.4",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.3.4",
-    "@fugood/node-llama-darwin-x64": "1.3.4",
-    "@fugood/node-llama-darwin-arm64": "1.3.4"
+    "@fugood/node-llama-linux-x64": "1.3.6",
+    "@fugood/node-llama-linux-x64-vulkan": "1.3.6",
+    "@fugood/node-llama-linux-x64-cuda": "1.3.6",
+    "@fugood/node-llama-linux-arm64-snapdragon": "1.3.6",
+    "@fugood/node-llama-linux-arm64": "1.3.6",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.3.6",
+    "@fugood/node-llama-linux-arm64-cuda": "1.3.6",
+    "@fugood/node-llama-win32-x64": "1.3.6",
+    "@fugood/node-llama-win32-x64-vulkan": "1.3.6",
+    "@fugood/node-llama-win32-x64-cuda": "1.3.6",
+    "@fugood/node-llama-win32-arm64": "1.3.6",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.3.6",
+    "@fugood/node-llama-darwin-x64": "1.3.6",
+    "@fugood/node-llama-darwin-arm64": "1.3.6"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/scripts/llama.cpp.patch

@@ -1,8 +1,8 @@
 diff --git a/src/llama.cpp/common/CMakeLists.txt b/src/llama.cpp/common/CMakeLists.txt
-index 706fa32ee..248459903 100644
+index bb168e835..cfc0e2c2e 100644
 --- a/src/llama.cpp/common/CMakeLists.txt
 +++ b/src/llama.cpp/common/CMakeLists.txt
-@@ -141,9 +141,16 @@ if (LLAMA_LLGUIDANCE)
+@@ -143,9 +143,16 @@ if (LLAMA_LLGUIDANCE)
      set(LLAMA_COMMON_EXTRA_LIBS ${LLAMA_COMMON_EXTRA_LIBS} llguidance ${LLGUIDANCE_PLATFORM_LIBS})
  endif ()
  
@@ -21,7 +21,7 @@ index 706fa32ee..248459903 100644
  
  #
 diff --git a/src/llama.cpp/common/chat.cpp b/src/llama.cpp/common/chat.cpp
-index 938872e82..6364f173f 100644
+index 6fa05a604..87dfa7a8b 100644
 --- a/src/llama.cpp/common/chat.cpp
 +++ b/src/llama.cpp/common/chat.cpp
 @@ -6,9 +6,6 @@
@@ -51,7 +51,7 @@ index 938872e82..6364f173f 100644
  struct templates_params {
      json messages;
      json tools;
-@@ -811,7 +798,7 @@ static std::string apply(
+@@ -817,7 +804,7 @@ static std::string apply(
          tmpl_inputs.extra_context.merge_patch(*additional_context);
      }
      // TODO: add flag to control date/time, if only for testing purposes.
@@ -61,7 +61,7 @@ index 938872e82..6364f173f 100644
      minja::chat_template_options tmpl_opts;
      // To avoid double BOS / EOS tokens, we're manually removing begining / trailing tokens
 diff --git a/src/llama.cpp/common/chat.h b/src/llama.cpp/common/chat.h
-index 50efb0d4e..f471a84c7 100644
+index 754c411e2..71241a6cc 100644
 --- a/src/llama.cpp/common/chat.h
 +++ b/src/llama.cpp/common/chat.h
 @@ -9,7 +9,18 @@
@@ -85,10 +85,10 @@ index 50efb0d4e..f471a84c7 100644
  struct common_chat_tool_call {
      std::string name;
 diff --git a/src/llama.cpp/common/common.cpp b/src/llama.cpp/common/common.cpp
-index 4dc95dcba..ea0ea86c0 100644
+index f3cc55247..65398844f 100644
 --- a/src/llama.cpp/common/common.cpp
 +++ b/src/llama.cpp/common/common.cpp
-@@ -1155,6 +1155,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
+@@ -1162,6 +1162,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
          mparams.n_gpu_layers = params.n_gpu_layers;
      }
  
@@ -97,10 +97,10 @@ index 4dc95dcba..ea0ea86c0 100644
      mparams.split_mode      = params.split_mode;
      mparams.tensor_split    = params.tensor_split;
 diff --git a/src/llama.cpp/common/common.h b/src/llama.cpp/common/common.h
-index f42c083fa..c573cc812 100644
+index de5b404dd..d30d252c9 100644
 --- a/src/llama.cpp/common/common.h
 +++ b/src/llama.cpp/common/common.h
-@@ -274,6 +274,7 @@ struct lr_opt {
+@@ -281,6 +281,7 @@ struct lr_opt {
  struct ggml_opt_optimizer_params common_opt_lr_pars(void * userdata);
  
  struct common_params {
@@ -109,7 +109,7 @@ index f42c083fa..c573cc812 100644
      int32_t n_ctx                 =  4096; // context size
      int32_t n_batch               =  2048; // logical batch size for prompt processing (must be >=32 to use BLAS)
 diff --git a/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt b/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
-index e52e050a8..c1000c162 100644
+index d0cab0bcb..48d532838 100644
 --- a/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
 +++ b/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt
 @@ -106,7 +106,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
@@ -121,6 +121,27 @@ index e52e050a8..c1000c162 100644
          else()
              check_cxx_compiler_flag(-mfp16-format=ieee GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E)
              if (NOT "${GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
+diff --git a/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+index cabd301ad..31eec134c 100644
+--- a/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp
++++ b/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+@@ -9,6 +9,7 @@
+ #include <chrono>
+ #include <mutex>
+ #include <string>
++#include <stdexcept>
+ 
+ #ifdef _WIN32
+ #    include <sal.h>
+@@ -3682,6 +3683,8 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
+         } catch (std::exception const &exc) {
+             GGML_LOG_ERROR("ggml-hex: failed to create device/session %zu\n", i);
+             devices[i].context = nullptr;
++            opt_ndev = i;
++            break;
+         }
+     }
+ }
 diff --git a/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt b/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt
 index de01336cd..29b1a043d 100644
 --- a/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt

package/src/LlamaContext.cpp

@@ -105,6 +105,9 @@ void LlamaContext::Init(Napi::Env env, Napi::Object &exports) {
        InstanceMethod<&LlamaContext::GetModelInfo>(
            "getModelInfo",
            static_cast<napi_property_attributes>(napi_enumerable)),
+       InstanceMethod<&LlamaContext::GetUsedDevices>(
+           "getUsedDevices",
+           static_cast<napi_property_attributes>(napi_enumerable)),
        InstanceMethod<&LlamaContext::GetFormattedChat>(
            "getFormattedChat",
            static_cast<napi_property_attributes>(napi_enumerable)),
@@ -303,9 +306,32 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
     params.tensor_buft_overrides.push_back({nullptr, nullptr});
   }
 
+  auto cpu_mask = get_option<std::string>(options, "cpu_mask", "");
+  if (!cpu_mask.empty()) {
+    params.cpuparams.mask_valid = true;
+    if (!parse_cpu_mask(cpu_mask, params.cpuparams.cpumask)) {
+      Napi::TypeError::New(env, "Invalid cpu_mask").ThrowAsJavaScriptException();
+    }
+  }
+
+  params.cpuparams.strict_cpu = get_option<bool>(options, "cpu_strict", false);
+
   llama_backend_init();
   llama_numa_init(params.numa);
 
+  // Parse devices array
+  if (options.Has("devices") && options.Get("devices").IsArray()) {
+    auto devices_array = options.Get("devices").As<Napi::Array>();
+    for (size_t i = 0; i < devices_array.Length(); i++) {
+      auto device_name = devices_array.Get(i).ToString().Utf8Value();
+      auto * dev = ggml_backend_dev_by_name(device_name.c_str());
+      if (dev) {
+        params.devices.push_back(dev);
+      }
+      // Skip invalid device names silently
+    }
+  }
+
   std::vector<common_adapter_lora_info> lora;
   auto lora_path = get_option<std::string>(options, "lora", "");
   auto lora_scaled = get_option<float>(options, "lora_scaled", 1.0f);
@@ -378,6 +404,17 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
   }
   _rn_ctx->attachThreadpoolsIfAvailable();
 
+  // Collect used devices from the loaded model
+  if (_rn_ctx->llama_init.model) {
+    const auto &model_devices = _rn_ctx->llama_init.model->devices;
+    for (auto dev : model_devices) {
+      const char *dev_name = ggml_backend_dev_name(dev);
+      if (dev_name != nullptr) {
+        _used_devices.push_back(std::string(dev_name));
+      }
+    }
+  }
+
   // Release progress callback after model is loaded
   if (has_progress_callback) {
     _progress_tsfn.Release();
@@ -583,6 +620,15 @@ Napi::Value LlamaContext::GetModelInfo(const Napi::CallbackInfo &info) {
   return details;
 }
 
+// getUsedDevices(): string[]
+Napi::Value LlamaContext::GetUsedDevices(const Napi::CallbackInfo &info) {
+  Napi::Env env = info.Env();
+  Napi::Array devices = Napi::Array::New(env, _used_devices.size());
+  for (size_t i = 0; i < _used_devices.size(); i++) {
+    devices[i] = Napi::String::New(env, _used_devices[i]);
+  }
+  return devices;
+}
 
 
 // getFormattedChat(

package/src/LlamaContext.h

@@ -31,6 +31,7 @@ public:
 private:
   Napi::Value GetSystemInfo(const Napi::CallbackInfo &info);
   Napi::Value GetModelInfo(const Napi::CallbackInfo &info);
+  Napi::Value GetUsedDevices(const Napi::CallbackInfo &info);
   Napi::Value GetFormattedChat(const Napi::CallbackInfo &info);
   Napi::Value Completion(const Napi::CallbackInfo &info);
   void StopCompletion(const Napi::CallbackInfo &info);
@@ -69,6 +70,7 @@ private:
   void CancelRequest(const Napi::CallbackInfo &info);
 
   std::string _info;
+  std::vector<std::string> _used_devices;
   Napi::Object _meta;
   LlamaCompletionWorker *_wip = nullptr;
 

package/src/llama.cpp/common/CMakeLists.txt

@@ -50,6 +50,8 @@ add_library(${TARGET} STATIC
     base64.hpp
     chat-parser.cpp
     chat-parser.h
+    chat-parser-xml-toolcall.h
+    chat-parser-xml-toolcall.cpp
     chat.cpp
     chat.h
     common.cpp