@novastera-oss/llamarn 0.5.4 → 0.6.1

package/README.md

@@ -13,7 +13,8 @@
 
 * Basic model loading and inference
 * Metal support on iOS
-* OpenCL/Vulkan support on Android (in progress)
+* OpenCL/Vulkan GPU acceleration on Android
+* Snapdragon Hexagon NPU support on Android (arm64-v8a)
 * Automatic CPU/GPU detection
 * Chat completion with templates (including Jinja template support)
 * Embeddings generation
@@ -26,15 +27,17 @@
 
 We welcome contributions, especially in these areas:
 
-1. **Android GPU Testing and Detection**:
-   * Development of reliable GPU detection mechanism in React Native
+1. **Android GPU and NPU Testing**:
+   * **OpenCL/Vulkan GPU Libraries**: GPU acceleration libraries (OpenCL and Vulkan) have been built and integrated, but we need help testing them on various Android devices to ensure proper functionality and performance.
+   * **Snapdragon Hexagon NPU Support**: Hexagon NPU support has been added for Snapdragon devices (arm64-v8a), but we need community testing on actual Snapdragon devices to verify it works correctly.
+   * Development of reliable GPU/NPU detection mechanism in React Native
    * Implementation of proper backend initialization verification
-   * Creation of robust testing framework for GPU availability
-   * Integration of OpenCL and Vulkan acceleration once detection is stable
-   * Performance benchmarking and optimization for mobile GPUs
+   * Creation of robust testing framework for GPU/NPU availability
+   * Performance benchmarking and optimization for mobile GPUs and NPUs
+   * Real-world device testing across different manufacturers and chipset generations
 
 2. **CI Improvements**:
-   * Adding automated Android GPU tests to CI pipeline
+   * Adding automated Android GPU/NPU tests to CI pipeline
    * Implementing device-specific testing strategies
    * Adding performance benchmarks to CI
 

package/android/CMakeLists.txt

@@ -269,6 +269,27 @@ if(EXISTS ${JNI_LIBS_DIR}/${ANDROID_ABI}/libggml-opencl.so)
     )
 endif()
 
+# Copy Hexagon backend library if it exists (Snapdragon devices only - arm64-v8a)
+if(EXISTS ${JNI_LIBS_DIR}/${ANDROID_ABI}/libggml-hexagon.so)
+    add_custom_command(TARGET RNLlamaCpp POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+            ${JNI_LIBS_DIR}/${ANDROID_ABI}/libggml-hexagon.so
+            $<TARGET_FILE_DIR:RNLlamaCpp>/libggml-hexagon.so
+        COMMENT "Copying Hexagon backend library to build output directory"
+    )
+    # Also copy all HTP libraries (required for Hexagon backend)
+    file(GLOB HTP_LIBS "${JNI_LIBS_DIR}/${ANDROID_ABI}/libggml-htp-*.so")
+    foreach(HTP_LIB ${HTP_LIBS})
+        get_filename_component(HTP_LIB_NAME ${HTP_LIB} NAME)
+        add_custom_command(TARGET RNLlamaCpp POST_BUILD
+            COMMAND ${CMAKE_COMMAND} -E copy_if_different
+                ${HTP_LIB}
+                $<TARGET_FILE_DIR:RNLlamaCpp>/${HTP_LIB_NAME}
+            COMMENT "Copying Hexagon HTP library ${HTP_LIB_NAME} to build output directory"
+        )
+    endforeach()
+endif()
+
 # NOTE: We do NOT copy libOpenCL.so or libvulkan.so - these are system libraries
 # The ICD loader is built and installed to NDK sysroot for BUILD-TIME linking only.
 # At runtime, the system will provide libOpenCL.so and libvulkan.so if the device supports them.

package/android/gradle.properties

@@ -2,4 +2,4 @@ RNLlamaCpp_kotlinVersion=2.0.21
 RNLlamaCpp_minSdkVersion=28
 RNLlamaCpp_targetSdkVersion=34
 RNLlamaCpp_compileSdkVersion=35
-RNLlamaCpp_ndkVersion=27.1.12297006
+RNLlamaCpp_ndkVersion=28.2.13676358

package/android/src/main/AndroidManifest.xml

@@ -8,8 +8,36 @@
     <uses-native-library
         android:name="libggml-vulkan.so"
         android:required="false" />
+    <uses-native-library
+        android:name="libggml-hexagon.so"
+        android:required="false" />
 
-    <!-- System loaders (only mapped if present on device) -->
+    <!-- Hexagon HTP (Hexagon Tensor Processor) libraries for Snapdragon NPU -->
+    <!-- Unlike OpenCL/Vulkan which have system libraries (libOpenCL.so, libvulkan.so), -->
+    <!-- Hexagon has no system library - all HTP libraries must be bundled with the app. -->
+    <!-- At runtime, libggml-hexagon.so queries the device hardware via FastRPC to detect -->
+    <!-- which Hexagon version the device has, then loads the corresponding libggml-htp-vXX.so. -->
+    <!-- Since we don't know the device at build time, all versions are shipped. -->
+    <!-- v73, v75, v79, v81 are commonly used on modern Snapdragon devices. -->
+    <uses-native-library
+        android:name="libggml-htp-v73.so"
+        android:required="false" />
+    <uses-native-library
+        android:name="libggml-htp-v75.so"
+        android:required="false" />
+    <uses-native-library
+        android:name="libggml-htp-v79.so"
+        android:required="false" />
+    <uses-native-library
+        android:name="libggml-htp-v81.so"
+        android:required="false" />
+
+    <!-- System libraries (provided by device/Snapdragon, not shipped with app) -->
+    <!-- libcdsprpc.so: FastRPC library for communicating with CDSP (Compute DSP) domain on Snapdragon -->
+    <!-- Required for Hexagon backend to communicate with the NPU -->
+    <uses-native-library
+        android:name="libcdsprpc.so"
+        android:required="false" />
     <uses-native-library
         android:name="libOpenCL.so"
         android:required="false" />

package/cpp/PureCppImpl.cpp

@@ -97,23 +97,44 @@ jsi::Value PureCppImpl::loadLlamaModelInfo(jsi::Runtime &runtime, jsi::String mo
           
           // Load all available backends (CPU is dynamically loaded when GGML_BACKEND_DL is enabled)
           // With GGML_BACKEND_DL=ON, ALL backends (CPU + GPU) are dynamically loaded
-          // CPU backend is in libggml-cpu.so, GPU backends are in libggml-opencl.so, libggml-vulkan.so
+          // CPU backend is in libggml-cpu.so, GPU backends are in libggml-opencl.so, libggml-vulkan.so, libggml-hexagon.so
           // On Android, dlopen() can load libraries by name even from inside APKs
           #ifdef __ANDROID__
           // Load CPU backend directly - Android's linker will find it in the same directory
-          void* cpu_handle = dlopen("libggml-cpu.so", RTLD_LAZY | RTLD_LOCAL);
-          if (cpu_handle) {
-            typedef ggml_backend_reg_t (*backend_init_fn_t)();
-            backend_init_fn_t backend_init = (backend_init_fn_t)dlsym(cpu_handle, "ggml_backend_init");
-            if (backend_init) {
-              ggml_backend_reg_t cpu_backend = backend_init();
-              if (cpu_backend) {
-                ggml_backend_register(cpu_backend);
+          // Check if already registered to avoid duplicate registration
+          if (!ggml_backend_reg_by_name("CPU")) {
+            void* cpu_handle = dlopen("libggml-cpu.so", RTLD_LAZY | RTLD_LOCAL);
+            if (cpu_handle) {
+              typedef ggml_backend_reg_t (*backend_init_fn_t)();
+              backend_init_fn_t backend_init = (backend_init_fn_t)dlsym(cpu_handle, "ggml_backend_init");
+              if (backend_init) {
+                ggml_backend_reg_t cpu_backend = backend_init();
+                if (cpu_backend) {
+                  ggml_backend_register(cpu_backend);
+                }
               }
             }
           }
           
-          // Load GPU backends (OpenCL, Vulkan) if present - they will be found by name
+          // Load Hexagon backend (Snapdragon DSP) - more performant than Vulkan on Snapdragon devices
+          // Load before other GPU backends to give it priority
+          // Check if already registered to avoid duplicate registration
+          if (!ggml_backend_reg_by_name("HTP")) {
+            void* hexagon_handle = dlopen("libggml-hexagon.so", RTLD_LAZY | RTLD_LOCAL);
+            if (hexagon_handle) {
+              typedef ggml_backend_reg_t (*backend_init_fn_t)();
+              backend_init_fn_t backend_init = (backend_init_fn_t)dlsym(hexagon_handle, "ggml_backend_init");
+              if (backend_init) {
+                ggml_backend_reg_t hexagon_backend = backend_init();
+                if (hexagon_backend) {
+                  ggml_backend_register(hexagon_backend);
+                }
+              }
+            }
+          }
+          
+          // Load other GPU backends (OpenCL, Vulkan) if present - they will be found by name
+          // ggml_backend_load_all() will skip backends that are already loaded
           ggml_backend_load_all();
           #else
           ggml_backend_load_all();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@novastera-oss/llamarn",
-  "version": "0.5.4",
+  "version": "0.6.1",
   "description": "An attempt at a pure cpp turbo module library",
   "source": "./src/index.tsx",
   "main": "./lib/module/index.js",