@runanywhere/llamacpp 0.17.6 → 0.18.0

package/README.md

@@ -10,6 +10,8 @@ LlamaCPP backend for the RunAnywhere React Native SDK. Provides on-device LLM te
 
 - **Text Generation** — Generate text responses from prompts
 - **Streaming** — Real-time token-by-token output
+- **Tool Calling** — Let models invoke registered tools during generation
+- **Structured Output** — Generate type-safe JSON responses
 - **GGUF Support** — Run any GGUF-format model (Llama, Mistral, Qwen, SmolLM, etc.)
 - **Metal GPU Acceleration** — 3-5x faster inference on Apple Silicon (iOS)
 - **CPU Inference** — Works on all devices without GPU requirements
@@ -20,7 +22,7 @@ LlamaCPP backend for the RunAnywhere React Native SDK. Provides on-device LLM te
 ## Requirements
 
 - `@runanywhere/core` (peer dependency)
-- React Native 0.71+
+- React Native 0.74+
 - iOS 15.1+ / Android API 24+
 
 ---
@@ -246,6 +248,51 @@ const result = await streamResult.result;
 console.log('\nSpeed:', result.performanceMetrics.tokensPerSecond, 'tok/s');
 ```
 
+#### Tool Calling
+
+Register tools and let the LLM call them during generation. Tool calling parsing and prompt formatting is handled entirely in C++ for consistency across platforms.
+
+```typescript
+import { RunAnywhere } from '@runanywhere/core';
+import { LlamaCPP } from '@runanywhere/llamacpp';
+
+// Register a tool
+RunAnywhere.registerTool(
+  {
+    name: 'calculate',
+    description: 'Perform a math calculation',
+    parameters: [
+      { name: 'expression', type: 'string', description: 'Math expression', required: true },
+    ],
+  },
+  async (args) => {
+    const result = eval(args.expression as string); // simplified example
+    return { result };
+  }
+);
+
+// Generate with tools
+const result = await RunAnywhere.generateWithTools(
+  'What is 42 * 17?',
+  {
+    autoExecute: true,
+    maxToolCalls: 3,
+    temperature: 0.7,
+    format: 'default', // 'default' for most models, 'lfm2' for Liquid AI models
+  }
+);
+console.log(result.text); // "42 * 17 = 714"
+```
+
+**Supported tool calling formats:**
+
+| Format | Tag Pattern | Models |
+|--------|-------------|--------|
+| `default` | `<tool_call>{"tool":"name","arguments":{}}</tool_call>` | Llama, Qwen, Mistral, SmolLM, most GGUF models |
+| `lfm2` | `<\|tool_call_start\|>[func(arg="val")]<\|tool_call_end\|>` | Liquid AI LFM2-Tool models |
+
+---
+
 #### Model Management
 
 ```typescript
@@ -270,27 +317,38 @@ Any GGUF-format model works with this backend. Recommended models:
 
 ### Small Models (< 1GB RAM)
 
-| Model | Size | Memory | Description |
-|-------|------|--------|-------------|
-| SmolLM2 360M Q8_0 | ~400MB | 500MB | Fast, lightweight |
-| Qwen 2.5 0.5B Q6_K | ~500MB | 600MB | Multilingual |
-| LFM2 350M Q4_K_M | ~200MB | 250MB | Ultra-compact |
+| Model | Size | Memory | Tool Calling | Description |
+|-------|------|--------|:------------:|-------------|
+| SmolLM2 360M Q8_0 | ~400MB | 500MB | - | Fast, lightweight |
+| Qwen 2.5 0.5B Q6_K | ~500MB | 600MB | Yes | Multilingual |
+| LFM2 350M Q4_K_M | ~200MB | 250MB | Yes (lfm2) | Ultra-compact, Liquid AI |
 
 ### Medium Models (1-3GB RAM)
 
-| Model | Size | Memory | Description |
-|-------|------|--------|-------------|
-| Phi-3 Mini Q4_K_M | ~2GB | 2.5GB | Microsoft |
-| Gemma 2B Q4_K_M | ~1.5GB | 2GB | Google |
-| TinyLlama 1.1B Q4_K_M | ~700MB | 1GB | Fast chat |
+| Model | Size | Memory | Tool Calling | Description |
+|-------|------|--------|:------------:|-------------|
+| Phi-3 Mini Q4_K_M | ~2GB | 2.5GB | - | Microsoft |
+| Gemma 2B Q4_K_M | ~1.5GB | 2GB | - | Google |
+| LFM2 1.2B Q4_K_M | ~800MB | 1GB | Yes (lfm2) | Liquid AI tool-calling |
+| Qwen 2.5 1.5B Instruct Q4_K_M | ~1GB | 1.5GB | Yes | Alibaba, multilingual |
+| TinyLlama 1.1B Q4_K_M | ~700MB | 1GB | - | Fast chat |
 
 ### Large Models (4GB+ RAM)
 
-| Model | Size | Memory | Description |
-|-------|------|--------|-------------|
-| Llama 2 7B Q4_K_M | ~4GB | 5GB | Meta |
-| Mistral 7B Q4_K_M | ~4GB | 5GB | Mistral AI |
-| Llama 3.2 3B Q4_K_M | ~2GB | 3GB | Meta latest |
+| Model | Size | Memory | Tool Calling | Description |
+|-------|------|--------|:------------:|-------------|
+| Llama 3.2 3B Instruct Q4_K_M | ~2GB | 3GB | Yes | Meta latest |
+| Mistral 7B Instruct Q4_K_M | ~4GB | 5GB | Yes | Mistral AI |
+| Qwen 2.5 7B Instruct Q4_K_M | ~4GB | 5GB | Yes | Alibaba |
+| Llama 2 7B Chat Q4_K_M | ~4GB | 5GB | - | Meta |
+
+### Tool Calling Model Selection Guide
+
+- **Best for tool calling (small):** LFM2-350M-Tool (use `format: 'lfm2'`) or Qwen 2.5 0.5B
+- **Best for tool calling (medium):** LFM2-1.2B-Tool or Qwen 2.5 1.5B Instruct
+- **Best for tool calling (large):** Mistral 7B Instruct or Qwen 2.5 7B Instruct
+- **Instruct-tuned models** generally perform better at following tool calling instructions
+- Use `format: 'lfm2'` only with Liquid AI LFM2-Tool models; all others use `format: 'default'`
 
 ---
 

package/android/CMakeLists.txt

@@ -19,8 +19,12 @@ set(JNILIB_DIR ${CMAKE_SOURCE_DIR}/src/main/jniLibs/${ANDROID_ABI})
 # Downloaded via Gradle downloadNativeLibs task
 # =============================================================================
 if(NOT EXISTS "${JNILIB_DIR}/librac_backend_llamacpp.so")
-    message(FATAL_ERROR "[RunAnywhereLlama] RABackendLlamaCPP not found at ${JNILIB_DIR}/librac_backend_llamacpp.so\n"
-                        "Run: ./gradlew :runanywhere_llamacpp:downloadNativeLibs")
+    message(WARNING "[RunAnywhereLlama] RABackendLlamaCPP not found for ${ANDROID_ABI} at ${JNILIB_DIR}/librac_backend_llamacpp.so\n"
+                    "This ABI will not be functional. To fix, run: ./gradlew :runanywhere_llamacpp:downloadNativeLibs\n"
+                    "Or set reactNativeArchitectures=arm64-v8a in gradle.properties to skip this ABI.")
+    file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/stub.cpp" "// Stub for missing ABI ${ANDROID_ABI}")
+    add_library(${PACKAGE_NAME} SHARED "${CMAKE_CURRENT_BINARY_DIR}/stub.cpp")
+    return()
 endif()
 
 add_library(rac_backend_llamacpp SHARED IMPORTED)

package/android/build.gradle

@@ -38,9 +38,13 @@ def getExtOrDefault(name) {
     return rootProject.ext.has(name) ? rootProject.ext.get(name) : project.properties['RunAnywhereLlama_' + name]
 }
 
-// Only arm64-v8a is supported
+// Supported ABIs - arm64-v8a for physical devices, x86_64 for emulators
+// Can be overridden via gradle.properties: reactNativeArchitectures=arm64-v8a
 def reactNativeArchitectures() {
-    return ["arm64-v8a"]
+    def value = rootProject.hasProperty("reactNativeArchitectures")
+        ? rootProject.property("reactNativeArchitectures")
+        : null
+    return value ? value.split(",").collect { it.trim() } : ["arm64-v8a", "x86_64"]
 }
 
 apply plugin: 'com.android.library'
@@ -109,7 +113,7 @@ android {
         targetSdkVersion getExtOrIntegerDefault('targetSdkVersion')
 
         ndk {
-            abiFilters 'arm64-v8a'
+            abiFilters(*reactNativeArchitectures())
         }
 
         externalNativeBuild {
@@ -118,7 +122,7 @@ android {
                 arguments "-DANDROID_STL=c++_shared",
                           // Fix NitroModules prefab path - use app's build directory
                           "-DREACT_NATIVE_NITRO_BUILD_DIR=${rootProject.buildDir}"
-                abiFilters 'arm64-v8a'
+                abiFilters(*reactNativeArchitectures())
             }
         }
     }
@@ -132,7 +136,12 @@ android {
     packagingOptions {
         excludes = [
             "META-INF",
-            "META-INF/**"
+            "META-INF/**",
+            // Exclude librac_commons.so from this module's packaging.
+            // The core package (@runanywhere/core) is the single authoritative source
+            // for librac_commons.so. If this module also packages it, Gradle's native
+            // lib merge may pick a stale version, causing UnsatisfiedLinkError crashes.
+            "**/librac_commons.so"
         ]
         pickFirsts = [
             "**/libc++_shared.so",
@@ -202,11 +211,64 @@ task downloadNativeLibs {
             return
         }
 
-        // Check if libs are already bundled (npm install case)
-        def bundledLibsDir = file("${jniLibsDir}/arm64-v8a")
-        def bundledLibs = bundledLibsDir.exists() ? bundledLibsDir.listFiles()?.findAll { it.name.endsWith(".so") } : []
-        if (bundledLibs?.size() > 0) {
-            logger.lifecycle("[RunAnywhereLlama] ✅ Using bundled native libraries from npm package (${bundledLibs.size()} .so files)")
+        // Check if libs are already bundled for ALL requested ABIs (npm install case)
+        def requestedAbis = reactNativeArchitectures()
+        def allAbisBundled = requestedAbis.every { abi ->
+            def abiDir = file("${jniLibsDir}/${abi}")
+            def libs = abiDir.exists() ? abiDir.listFiles()?.findAll { it.name.endsWith(".so") } : []
+            return libs?.size() > 0
+        }
+        if (allAbisBundled) {
+            logger.lifecycle("[RunAnywhereLlama] ✅ Using bundled native libraries from npm package for ABIs: ${requestedAbis.join(', ')}")
+            return
+        }
+        // Check if at least arm64-v8a is bundled (partial bundle - need to download missing ABIs)
+        def arm64Dir = file("${jniLibsDir}/arm64-v8a")
+        def arm64Bundled = arm64Dir.exists() && arm64Dir.listFiles()?.any { it.name.endsWith(".so") }
+        if (arm64Bundled) {
+            def missingAbis = requestedAbis.findAll { abi ->
+                def abiDir = file("${jniLibsDir}/${abi}")
+                def libs = abiDir.exists() ? abiDir.listFiles()?.findAll { it.name.endsWith(".so") } : []
+                return !libs || libs.size() == 0
+            }
+            if (missingAbis.size() > 0) {
+                logger.lifecycle("[RunAnywhereLlama] ⚠️ Bundled libs found for arm64-v8a but missing for: ${missingAbis.join(', ')}")
+                logger.lifecycle("[RunAnywhereLlama] Attempting to download missing ABIs from GitHub releases...")
+                try {
+                    def llamacppUrl = "https://github.com/${githubOrg}/${coreRepo}/releases/download/core-v${coreVersion}/RABackendLlamaCPP-android-v${coreVersion}.zip"
+                    def tempZip = file("${downloadedLibsDir}/RABackendLlamaCPP-supplement.zip")
+                    downloadedLibsDir.mkdirs()
+                    new URL(llamacppUrl).withInputStream { input ->
+                        tempZip.withOutputStream { output -> output << input }
+                    }
+                    copy {
+                        from zipTree(tempZip)
+                        into jniLibsDir
+                        exclude "**/libc++_shared.so"
+                        // Exclude librac_commons.so - the core package (@runanywhere/core) is the
+                        // authoritative source. Including it here risks a stale version winning the
+                        // Gradle native lib merge, causing UnsatisfiedLinkError crashes at runtime.
+                        exclude "**/librac_commons.so"
+                        eachFile { fileCopyDetails ->
+                            def pathString = fileCopyDetails.relativePath.pathString
+                            def match = pathString =~ /.*\/(arm64-v8a|armeabi-v7a|x86|x86_64)\/(.+\.so)$/
+                            if (match) {
+                                def abi = match[0][1]
+                                def filename = match[0][2]
+                                fileCopyDetails.relativePath = new RelativePath(true, abi, filename)
+                            } else if (!pathString.endsWith(".so")) {
+                                fileCopyDetails.exclude()
+                            }
+                        }
+                        includeEmptyDirs = false
+                    }
+                    tempZip.delete()
+                    logger.lifecycle("[RunAnywhereLlama] ✅ Downloaded missing ABIs successfully")
+                } catch (Exception e) {
+                    logger.warn("[RunAnywhereLlama] ⚠️ Could not download missing ABIs: ${e.message}")
+                    logger.warn("[RunAnywhereLlama] Building with available ABIs only (arm64-v8a)")
+                }
+            }
             return
         }
 
@@ -249,6 +311,10 @@ task downloadNativeLibs {
                 // IMPORTANT: Exclude libc++_shared.so - React Native provides its own
                 // Using a different version causes ABI compatibility issues
                 exclude "**/libc++_shared.so"
+                // Exclude librac_commons.so - the core package (@runanywhere/core) is the
+                // authoritative source. Including it here risks a stale version winning the
+                // Gradle native lib merge, causing UnsatisfiedLinkError crashes at runtime.
+                exclude "**/librac_commons.so"
                 eachFile { fileCopyDetails ->
                     def pathString = fileCopyDetails.relativePath.pathString
                     // Handle RABackendLlamaCPP-android-vX.Y.Z/llamacpp/ABI/*.so structure

package/ios/Frameworks/RABackendLLAMACPP.xcframework/Info.plist

@@ -8,32 +8,32 @@
 			<key>BinaryPath</key>
 			<string>RABackendLLAMACPP.framework/RABackendLLAMACPP</string>
 			<key>LibraryIdentifier</key>
-			<string>ios-arm64</string>
+			<string>ios-arm64_x86_64-simulator</string>
 			<key>LibraryPath</key>
 			<string>RABackendLLAMACPP.framework</string>
 			<key>SupportedArchitectures</key>
 			<array>
 				<string>arm64</string>
+				<string>x86_64</string>
 			</array>
 			<key>SupportedPlatform</key>
 			<string>ios</string>
+			<key>SupportedPlatformVariant</key>
+			<string>simulator</string>
 		</dict>
 		<dict>
 			<key>BinaryPath</key>
 			<string>RABackendLLAMACPP.framework/RABackendLLAMACPP</string>
 			<key>LibraryIdentifier</key>
-			<string>ios-arm64_x86_64-simulator</string>
+			<string>ios-arm64</string>
 			<key>LibraryPath</key>
 			<string>RABackendLLAMACPP.framework</string>
 			<key>SupportedArchitectures</key>
 			<array>
 				<string>arm64</string>
-				<string>x86_64</string>
 			</array>
 			<key>SupportedPlatform</key>
 			<string>ios</string>
-			<key>SupportedPlatformVariant</key>
-			<string>simulator</string>
 		</dict>
 	</array>
 	<key>CFBundlePackageType</key>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@runanywhere/llamacpp",
-  "version": "0.17.6",
+  "version": "0.18.0",
   "description": "LlamaCpp backend for RunAnywhere React Native SDK - GGUF model support for on-device LLM",
   "main": "src/index.ts",
   "types": "src/index.ts",
@@ -18,7 +18,10 @@
     "src",
     "cpp",
     "ios",
-    "android",
+    "!ios/build",
+    "android/src",
+    "android/build.gradle",
+    "android/CMakeLists.txt",
     "nitrogen",
     "nitro.json",
     "react-native.config.js",