@fugood/llama.node 1.3.5 → 1.3.6

package/CMakeLists.txt

@@ -44,7 +44,8 @@ else()
 endif()
 
 if (TO_PACKAGE)
-  set(PLATFORM_BINARY_DIR ${CMAKE_SOURCE_DIR}/packages/node-llama-${PLATFORM}-${ARCH}${VARIANT})
+  set(PACKAGE_NAME "node-llama-${PLATFORM}-${ARCH}${VARIANT}")
+  set(PLATFORM_BINARY_DIR ${CMAKE_SOURCE_DIR}/packages/${PACKAGE_NAME})
 else()
   set(PLATFORM_BINARY_DIR ${CMAKE_SOURCE_DIR}/build/Release)
 endif()
@@ -188,6 +189,13 @@ if (NOT MSVC AND CMAKE_SYSTEM_NAME STREQUAL "Windows")
   set(CMAKE_JS_LIB win_dynamic_load)
 endif()
 
+if (TO_PACKAGE AND GGML_HEXAGON)
+  set(NODE_RPATH "node_modules/@fugood/${PACKAGE_NAME}")
+  set(ELECTRON_ASAR_RPATH "resources/app.asar.unpacked/node_modules/@fugood/${PACKAGE_NAME}")
+  set(ELECTRON_RES_RPATH "resources/node_modules/@fugood/${PACKAGE_NAME}")
+  set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-rpath,${NODE_RPATH} -Wl,-rpath,${ELECTRON_ASAR_RPATH} -Wl,-rpath,${ELECTRON_RES_RPATH}")
+endif()
+
 add_library(${PROJECT_NAME} SHARED ${SOURCE_FILES} ${CMAKE_JS_SRC})
 set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "" SUFFIX ".node")
 target_link_libraries(${PROJECT_NAME} ${CMAKE_JS_LIB} llama ggml common mtmd ${CMAKE_THREAD_LIBS_INIT})
@@ -237,3 +245,15 @@ if (GGML_CLBLAST AND TO_PACKAGE)
     )
   endif()
 endif()
+
+if (GGML_HEXAGON)
+  get_target_property(HTP_LIBS_DIR ggml-hexagon BINARY_DIR)
+  add_custom_command(
+    TARGET copy_assets
+    COMMAND ${CMAKE_COMMAND} -E copy ${HTP_LIBS_DIR}/libggml-htp-v73.so ${PLATFORM_BINARY_DIR}
+    COMMAND ${CMAKE_COMMAND} -E copy ${HTP_LIBS_DIR}/libggml-htp-v75.so ${PLATFORM_BINARY_DIR}
+    COMMAND ${CMAKE_COMMAND} -E copy ${HTP_LIBS_DIR}/libggml-htp-v79.so ${PLATFORM_BINARY_DIR}
+    COMMAND ${CMAKE_COMMAND} -E copy ${HTP_LIBS_DIR}/libggml-htp-v81.so ${PLATFORM_BINARY_DIR}
+    COMMENT "Copying HTP libraries to bin folder"
+  )
+endif()

package/lib/binding.ts

@@ -25,6 +25,12 @@ export type LlamaModelOptions = {
   n_ctx?: number
   n_batch?: number
   n_ubatch?: number
+  /**
+   * CPU affinity mask
+   * Example: '0xfc'
+   */
+  cpu_mask?: string
+  cpu_strict?: boolean
   /**
    * Number of parallel sequences to support (sets n_seq_max).
    * This determines the maximum number of parallel slots that can be used.
@@ -574,7 +580,7 @@ export interface Module {
   LlamaContext: LlamaContext
 }
 
-export type LibVariant = 'default' | 'vulkan' | 'cuda'
+export type LibVariant = 'default' | 'vulkan' | 'cuda' | 'snapdragon'
 
 const getPlatformPackageName = (variant?: LibVariant): string => {
   const platform = process.platform

package/lib/index.js

@@ -201,7 +201,28 @@ const loadModel = (options, onProgress) => __awaiter(void 0, void 0, void 0, fun
     const variant = (_a = options.lib_variant) !== null && _a !== void 0 ? _a : 'default';
     (_b = mods[variant]) !== null && _b !== void 0 ? _b : (mods[variant] = yield (0, binding_1.loadModule)(options.lib_variant));
     refreshNativeLogSetup();
-    const nativeCtx = new mods[variant].LlamaContext(options, onProgress);
+    const { devices } = options;
+    let filteredDevs = [];
+    if (Array.isArray(devices)) {
+        filteredDevs = [...devices];
+        // Handle HTP* to use all HTP devices on Hexagon
+        if (variant === 'snapdragon' && devices.includes('HTP*')) {
+            const backendDevices = yield (0, exports.getBackendDevicesInfo)(variant);
+            const htpDevices = backendDevices
+                .filter((d) => d.deviceName.startsWith('HTP'))
+                .map((d) => d.deviceName);
+            filteredDevs = filteredDevs.reduce((acc, dev) => {
+                if (dev.startsWith('HTP*')) {
+                    acc.push(...htpDevices);
+                }
+                else if (!dev.startsWith('HTP')) {
+                    acc.push(dev);
+                }
+                return acc;
+            }, []);
+        }
+    }
+    const nativeCtx = new mods[variant].LlamaContext(Object.assign(Object.assign({}, options), { devices: filteredDevs.length > 0 ? filteredDevs : undefined }), onProgress);
     return new LlamaContextWrapper(nativeCtx);
 });
 exports.loadModel = loadModel;

package/lib/index.ts

@@ -309,7 +309,35 @@ export const loadModel = async (
   mods[variant] ??= await loadModule(options.lib_variant)
   refreshNativeLogSetup()
 
-  const nativeCtx = new mods[variant].LlamaContext(options, onProgress)
+  const { devices } = options
+  let filteredDevs: Array<string> = []
+  if (Array.isArray(devices)) {
+    filteredDevs = [...devices]
+
+    // Handle HTP* to use all HTP devices on Hexagon
+    if (variant === 'snapdragon' && devices.includes('HTP*')) {
+      const backendDevices = await getBackendDevicesInfo(variant)
+      const htpDevices = backendDevices
+        .filter((d) => d.deviceName.startsWith('HTP'))
+        .map((d) => d.deviceName)
+      filteredDevs = filteredDevs.reduce((acc, dev) => {
+        if (dev.startsWith('HTP*')) {
+          acc.push(...htpDevices)
+        } else if (!dev.startsWith('HTP')) {
+          acc.push(dev)
+        }
+        return acc
+      }, [] as Array<string>)
+    }
+  }
+
+  const nativeCtx = new mods[variant].LlamaContext(
+    {
+      ...options,
+      devices: filteredDevs.length > 0 ? filteredDevs : undefined,
+    },
+    onProgress,
+  )
   return new LlamaContextWrapper(nativeCtx)
 }
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fugood/llama.node",
   "access": "public",
-  "version": "1.3.5",
+  "version": "1.3.6",
   "description": "An another Node binding of llama.cpp",
   "main": "lib/index.js",
   "scripts": {
@@ -72,19 +72,20 @@
     "CMakeLists.txt"
   ],
   "optionalDependencies": {
-    "@fugood/node-llama-linux-x64": "1.3.5",
-    "@fugood/node-llama-linux-x64-vulkan": "1.3.5",
-    "@fugood/node-llama-linux-x64-cuda": "1.3.5",
-    "@fugood/node-llama-linux-arm64": "1.3.5",
-    "@fugood/node-llama-linux-arm64-vulkan": "1.3.5",
-    "@fugood/node-llama-linux-arm64-cuda": "1.3.5",
-    "@fugood/node-llama-win32-x64": "1.3.5",
-    "@fugood/node-llama-win32-x64-vulkan": "1.3.5",
-    "@fugood/node-llama-win32-x64-cuda": "1.3.5",
-    "@fugood/node-llama-win32-arm64": "1.3.5",
-    "@fugood/node-llama-win32-arm64-vulkan": "1.3.5",
-    "@fugood/node-llama-darwin-x64": "1.3.5",
-    "@fugood/node-llama-darwin-arm64": "1.3.5"
+    "@fugood/node-llama-linux-x64": "1.3.6",
+    "@fugood/node-llama-linux-x64-vulkan": "1.3.6",
+    "@fugood/node-llama-linux-x64-cuda": "1.3.6",
+    "@fugood/node-llama-linux-arm64-snapdragon": "1.3.6",
+    "@fugood/node-llama-linux-arm64": "1.3.6",
+    "@fugood/node-llama-linux-arm64-vulkan": "1.3.6",
+    "@fugood/node-llama-linux-arm64-cuda": "1.3.6",
+    "@fugood/node-llama-win32-x64": "1.3.6",
+    "@fugood/node-llama-win32-x64-vulkan": "1.3.6",
+    "@fugood/node-llama-win32-x64-cuda": "1.3.6",
+    "@fugood/node-llama-win32-arm64": "1.3.6",
+    "@fugood/node-llama-win32-arm64-vulkan": "1.3.6",
+    "@fugood/node-llama-darwin-x64": "1.3.6",
+    "@fugood/node-llama-darwin-arm64": "1.3.6"
   },
   "devDependencies": {
     "@babel/preset-env": "^7.24.4",

package/scripts/llama.cpp.patch

@@ -121,6 +121,27 @@ index d0cab0bcb..48d532838 100644
          else()
              check_cxx_compiler_flag(-mfp16-format=ieee GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E)
              if (NOT "${GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
+diff --git a/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+index cabd301ad..31eec134c 100644
+--- a/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp
++++ b/src/llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+@@ -9,6 +9,7 @@
+ #include <chrono>
+ #include <mutex>
+ #include <string>
++#include <stdexcept>
+ 
+ #ifdef _WIN32
+ #    include <sal.h>
+@@ -3682,6 +3683,8 @@ ggml_hexagon_registry::ggml_hexagon_registry(ggml_backend_reg_t reg) {
+         } catch (std::exception const &exc) {
+             GGML_LOG_ERROR("ggml-hex: failed to create device/session %zu\n", i);
+             devices[i].context = nullptr;
++            opt_ndev = i;
++            break;
+         }
+     }
+ }
 diff --git a/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt b/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt
 index de01336cd..29b1a043d 100644
 --- a/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt

package/src/LlamaContext.cpp

@@ -306,6 +306,16 @@ LlamaContext::LlamaContext(const Napi::CallbackInfo &info)
     params.tensor_buft_overrides.push_back({nullptr, nullptr});
   }
 
+  auto cpu_mask = get_option<std::string>(options, "cpu_mask", "");
+  if (!cpu_mask.empty()) {
+    params.cpuparams.mask_valid = true;
+    if (!parse_cpu_mask(cpu_mask, params.cpuparams.cpumask)) {
+      Napi::TypeError::New(env, "Invalid cpu_mask").ThrowAsJavaScriptException();
+    }
+  }
+
+  params.cpuparams.strict_cpu = get_option<bool>(options, "cpu_strict", false);
+
   llama_backend_init();
   llama_numa_init(params.numa);