@runanywhere/llamacpp 0.16.0

package/cpp/bridges/LLMBridge.hpp

@@ -0,0 +1,109 @@
+/**
+ * @file LLMBridge.hpp
+ * @brief LLM capability bridge for React Native
+ *
+ * Matches Swift's CppBridge+LLM.swift pattern, providing:
+ * - Model lifecycle (load/unload)
+ * - Text generation (sync and streaming)
+ * - Cancellation support
+ *
+ * Aligned with rac_llm_component.h and rac_llm_types.h API.
+ * RACommons is REQUIRED - no stub implementations.
+ */
+
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <string>
+
+// RACommons LLM headers - REQUIRED (flat include paths)
+#include "rac_llm_component.h"
+#include "rac_llm_types.h"
+
+namespace runanywhere {
+namespace bridges {
+
+/**
+ * @brief LLM streaming callbacks
+ */
+struct LLMStreamCallbacks {
+    std::function<bool(const std::string&)> onToken;
+    std::function<void(const std::string&, int, double)> onComplete;
+    std::function<void(int, const std::string&)> onError;
+};
+
+/**
+ * @brief LLM generation options
+ */
+struct LLMOptions {
+    int maxTokens = 512;
+    double temperature = 0.7;
+    double topP = 0.9;
+    int topK = 40;
+    std::string systemPrompt;
+    std::string stopSequence;
+};
+
+/**
+ * @brief LLM generation result
+ */
+struct LLMResult {
+    std::string text;
+    int tokenCount = 0;
+    double durationMs = 0.0;
+    bool cancelled = false;
+};
+
+/**
+ * @brief LLM capability bridge singleton
+ *
+ * Matches CppBridge+LLM.swift API.
+ * NOTE: RACommons is REQUIRED. All methods will throw std::runtime_error if
+ * the underlying C API calls fail.
+ */
+class LLMBridge {
+public:
+    static LLMBridge& shared();
+
+    // Lifecycle
+    bool isLoaded() const;
+    std::string currentModelId() const;
+    /**
+     * Load an LLM model
+     * @param modelPath Path to the model file (.gguf)
+     * @param modelId Model identifier for telemetry (e.g., "smollm2-360m-q8_0")
+     * @param modelName Human-readable model name (e.g., "SmolLM2 360M Q8_0")
+     * @return RAC_SUCCESS or error code
+     */
+    rac_result_t loadModel(const std::string& modelPath,
+                           const std::string& modelId = "",
+                           const std::string& modelName = "");
+    rac_result_t unload();
+    void cleanup();
+    void cancel();
+    void destroy();
+
+    // Generation
+    LLMResult generate(const std::string& prompt, const LLMOptions& options);
+    void generateStream(const std::string& prompt, const LLMOptions& options,
+                       const LLMStreamCallbacks& callbacks);
+
+    // State
+    rac_lifecycle_state_t getState() const;
+
+private:
+    LLMBridge();
+    ~LLMBridge();
+
+    // Disable copy/move
+    LLMBridge(const LLMBridge&) = delete;
+    LLMBridge& operator=(const LLMBridge&) = delete;
+
+    rac_handle_t handle_ = nullptr;
+    std::string loadedModelId_;
+    bool cancellationRequested_ = false;
+};
+
+} // namespace bridges
+} // namespace runanywhere

package/cpp/bridges/StructuredOutputBridge.cpp

@@ -0,0 +1,151 @@
+/**
+ * @file StructuredOutputBridge.cpp
+ * @brief Structured Output bridge implementation
+ *
+ * Uses RACommons structured output API for prompt preparation and JSON extraction.
+ * Uses LLMBridge for actual text generation.
+ * RACommons is REQUIRED - no stub implementations.
+ */
+
+#include "StructuredOutputBridge.hpp"
+#include "LLMBridge.hpp"
+#include <stdexcept>
+#include <cstdlib> // For free()
+
+// Unified logging via rac_logger.h
+#include "rac_logger.h"
+
+// Log category for this module
+#define LOG_CATEGORY "LLM.StructuredOutput"
+
+namespace runanywhere {
+namespace bridges {
+
+StructuredOutputBridge& StructuredOutputBridge::shared() {
+    static StructuredOutputBridge instance;
+    return instance;
+}
+
+StructuredOutputResult StructuredOutputBridge::generate(
+    const std::string& prompt,
+    const std::string& schema,
+    const std::string& optionsJson
+) {
+    StructuredOutputResult result;
+
+    if (!LLMBridge::shared().isLoaded()) {
+        throw std::runtime_error("StructuredOutputBridge: LLM model not loaded. Call loadModel() first.");
+    }
+
+    // Prepare the prompt using RACommons structured output API
+    rac_structured_output_config_t config = RAC_STRUCTURED_OUTPUT_DEFAULT;
+    config.json_schema = schema.c_str();
+    config.include_schema_in_prompt = RAC_TRUE;
+
+    char* preparedPrompt = nullptr;
+    rac_result_t prepResult = rac_structured_output_prepare_prompt(
+        prompt.c_str(),
+        &config,
+        &preparedPrompt
+    );
+
+    std::string structuredPrompt;
+    if (prepResult == RAC_SUCCESS && preparedPrompt) {
+        structuredPrompt = preparedPrompt;
+        free(preparedPrompt);
+    } else {
+        // Fallback: Build prompt manually
+        RAC_LOG_DEBUG(LOG_CATEGORY, "Fallback to manual prompt preparation");
+        structuredPrompt =
+            "You must respond with valid JSON matching this schema:\n" +
+            schema + "\n\n" +
+            "User request: " + prompt + "\n\n" +
+            "Respond with valid JSON only, no other text:";
+    }
+
+    // Generate using LLMBridge
+    LLMOptions opts;
+    opts.maxTokens = 1024;
+    opts.temperature = 0.1;  // Lower temperature for structured output
+    // TODO: Parse optionsJson if provided
+
+    LLMResult llmResult;
+    try {
+        llmResult = LLMBridge::shared().generate(structuredPrompt, opts);
+    } catch (const std::runtime_error& e) {
+        throw std::runtime_error("StructuredOutputBridge: LLM generation failed: " + std::string(e.what()));
+    }
+
+    if (llmResult.text.empty()) {
+        throw std::runtime_error("StructuredOutputBridge: LLM generation returned empty text.");
+    }
+
+    // Extract JSON using RACommons API
+    char* extractedJson = nullptr;
+    size_t jsonLength = 0;
+    rac_result_t extractResult = rac_structured_output_extract_json(
+        llmResult.text.c_str(),
+        &extractedJson,
+        &jsonLength
+    );
+
+    if (extractResult == RAC_SUCCESS && extractedJson && jsonLength > 0) {
+        result.json = std::string(extractedJson, jsonLength);
+        result.success = true;
+        free(extractedJson);
+        RAC_LOG_INFO(LOG_CATEGORY, "Successfully extracted JSON (%zu bytes)", jsonLength);
+    } else {
+        // Fallback: Try manual extraction
+        RAC_LOG_DEBUG(LOG_CATEGORY, "Fallback to manual JSON extraction");
+
+        std::string text = llmResult.text;
+        size_t start = 0, end = 0;
+
+        // Try using RACommons to find JSON boundaries
+        if (rac_structured_output_find_complete_json(text.c_str(), &start, &end) == RAC_TRUE) {
+            result.json = text.substr(start, end - start);
+            result.success = true;
+        } else {
+            // Manual fallback
+            start = text.find('{');
+            end = text.rfind('}');
+
+            if (start != std::string::npos && end != std::string::npos && end > start) {
+                result.json = text.substr(start, end - start + 1);
+                result.success = true;
+            } else {
+                // Try array
+                start = text.find('[');
+                end = text.rfind(']');
+                if (start != std::string::npos && end != std::string::npos && end > start) {
+                    result.json = text.substr(start, end - start + 1);
+                    result.success = true;
+                } else {
+                    throw std::runtime_error("StructuredOutputBridge: Could not extract valid JSON from response: " + text);
+                }
+            }
+        }
+    }
+
+    // Validate the extracted JSON (optional but good for debugging)
+    if (result.success) {
+        rac_structured_output_validation_t validation = {};
+        rac_result_t valResult = rac_structured_output_validate(
+            result.json.c_str(),
+            &config,
+            &validation
+        );
+
+        if (valResult != RAC_SUCCESS || validation.is_valid != RAC_TRUE) {
+            RAC_LOG_WARNING(LOG_CATEGORY, "Extracted JSON failed validation");
+            // Don't throw - the JSON was extracted, just log warning
+        }
+
+        rac_structured_output_validation_free(&validation);
+    }
+
+    return result;
+}
+
+} // namespace bridges
+} // namespace runanywhere

package/cpp/bridges/StructuredOutputBridge.hpp

@@ -0,0 +1,66 @@
+/**
+ * @file StructuredOutputBridge.hpp
+ * @brief Structured Output bridge for React Native
+ *
+ * Matches Swift's RunAnywhere+StructuredOutput.swift pattern, providing:
+ * - JSON schema-guided generation
+ * - Structured output extraction
+ *
+ * Aligned with rac_llm_structured_output.h API.
+ * RACommons is REQUIRED - no stub implementations.
+ */
+
+#pragma once
+
+#include <string>
+
+// RACommons structured output header - REQUIRED (flat include paths)
+#include "rac_llm_structured_output.h"
+#include "rac_llm_types.h"
+
+namespace runanywhere {
+namespace bridges {
+
+/**
+ * @brief Structured output result
+ */
+struct StructuredOutputResult {
+    std::string json;
+    bool success = false;
+    std::string error;
+};
+
+/**
+ * @brief Structured Output bridge singleton
+ *
+ * Generates LLM output following a JSON schema.
+ * NOTE: RACommons is REQUIRED. All methods will throw std::runtime_error if
+ * the underlying C API calls fail.
+ */
+class StructuredOutputBridge {
+public:
+    static StructuredOutputBridge& shared();
+
+    /**
+     * Generate structured output following a JSON schema
+     * @param prompt User prompt
+     * @param schema JSON schema string
+     * @param optionsJson Generation options
+     * @return Structured output result
+     */
+    StructuredOutputResult generate(
+        const std::string& prompt,
+        const std::string& schema,
+        const std::string& optionsJson = ""
+    );
+
+private:
+    StructuredOutputBridge() = default;
+    ~StructuredOutputBridge() = default;
+
+    StructuredOutputBridge(const StructuredOutputBridge&) = delete;
+    StructuredOutputBridge& operator=(const StructuredOutputBridge&) = delete;
+};
+
+} // namespace bridges
+} // namespace runanywhere

package/cpp/rac_llm_llamacpp.h

@@ -0,0 +1,34 @@
+/**
+ * @file rac_llm_llamacpp.h
+ * @brief Backend registration API for LlamaCPP
+ *
+ * Forward declarations for LlamaCPP backend registration functions.
+ * These symbols are exported by RABackendLLAMACPP.xcframework.
+ */
+
+#ifndef RAC_LLM_LLAMACPP_H
+#define RAC_LLM_LLAMACPP_H
+
+#include "rac_types.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Register the LlamaCPP backend with the RACommons service registry.
+ * @return RAC_SUCCESS on success, RAC_ERROR_MODULE_ALREADY_REGISTERED if already registered
+ */
+rac_result_t rac_backend_llamacpp_register(void);
+
+/**
+ * Unregister the LlamaCPP backend from the RACommons service registry.
+ * @return RAC_SUCCESS on success
+ */
+rac_result_t rac_backend_llamacpp_unregister(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RAC_LLM_LLAMACPP_H */

package/ios/Frameworks/RABackendLLAMACPP.xcframework/Info.plist

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>AvailableLibraries</key>
+	<array>
+		<dict>
+			<key>BinaryPath</key>
+			<string>RABackendLLAMACPP.framework/RABackendLLAMACPP</string>
+			<key>LibraryIdentifier</key>
+			<string>ios-arm64</string>
+			<key>LibraryPath</key>
+			<string>RABackendLLAMACPP.framework</string>
+			<key>SupportedArchitectures</key>
+			<array>
+				<string>arm64</string>
+			</array>
+			<key>SupportedPlatform</key>
+			<string>ios</string>
+		</dict>
+		<dict>
+			<key>BinaryPath</key>
+			<string>RABackendLLAMACPP.framework/RABackendLLAMACPP</string>
+			<key>LibraryIdentifier</key>
+			<string>ios-arm64_x86_64-simulator</string>
+			<key>LibraryPath</key>
+			<string>RABackendLLAMACPP.framework</string>
+			<key>SupportedArchitectures</key>
+			<array>
+				<string>arm64</string>
+				<string>x86_64</string>
+			</array>
+			<key>SupportedPlatform</key>
+			<string>ios</string>
+			<key>SupportedPlatformVariant</key>
+			<string>simulator</string>
+		</dict>
+	</array>
+	<key>CFBundlePackageType</key>
+	<string>XFWK</string>
+	<key>XCFrameworkFormatVersion</key>
+	<string>1.0</string>
+</dict>
+</plist>

package/ios/Frameworks/RABackendLLAMACPP.xcframework/ios-arm64/RABackendLLAMACPP.framework/Headers/RABackendLLAMACPP.h

@@ -0,0 +1,2 @@
+// RABackendLLAMACPP
+#include "rac_llamacpp.h"

package/ios/Frameworks/RABackendLLAMACPP.xcframework/ios-arm64/RABackendLLAMACPP.framework/Info.plist

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>CFBundleExecutable</key><string>RABackendLLAMACPP</string>
+    <key>CFBundleIdentifier</key><string>ai.runanywhere.RABackendLLAMACPP</string>
+    <key>CFBundlePackageType</key><string>FMWK</string>
+    <key>CFBundleShortVersionString</key><string>0.1.5</string>
+    <key>MinimumOSVersion</key><string>13.0</string>
+</dict>
+</plist>

package/ios/Frameworks/RABackendLLAMACPP.xcframework/ios-arm64/RABackendLLAMACPP.framework/Modules/module.modulemap

@@ -0,0 +1,5 @@
+framework module RABackendLLAMACPP {
+    umbrella header "RABackendLLAMACPP.h"
+    export *
+    module * { export * }
+}

package/ios/Frameworks/RABackendLLAMACPP.xcframework/ios-arm64_x86_64-simulator/RABackendLLAMACPP.framework/Headers/RABackendLLAMACPP.h

@@ -0,0 +1,2 @@
+// RABackendLLAMACPP
+#include "rac_llamacpp.h"

package/ios/Frameworks/RABackendLLAMACPP.xcframework/ios-arm64_x86_64-simulator/RABackendLLAMACPP.framework/Info.plist

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>CFBundleExecutable</key><string>RABackendLLAMACPP</string>
+    <key>CFBundleIdentifier</key><string>ai.runanywhere.RABackendLLAMACPP</string>
+    <key>CFBundlePackageType</key><string>FMWK</string>
+    <key>CFBundleShortVersionString</key><string>0.1.5</string>
+    <key>MinimumOSVersion</key><string>13.0</string>
+</dict>
+</plist>

package/ios/Frameworks/RABackendLLAMACPP.xcframework/ios-arm64_x86_64-simulator/RABackendLLAMACPP.framework/Modules/module.modulemap

@@ -0,0 +1,5 @@
+framework module RABackendLLAMACPP {
+    umbrella header "RABackendLLAMACPP.h"
+    export *
+    module * { export * }
+}

package/ios/LlamaCPPBackend.podspec

@@ -0,0 +1,127 @@
+require "json"
+
+package = JSON.parse(File.read(File.join(__dir__, "..", "package.json")))
+
+# =============================================================================
+# Version Constants (MUST match Swift Package.swift)
+# =============================================================================
+CORE_VERSION = "0.1.4"
+
+# =============================================================================
+# Binary Source - RABackendLlamaCPP from runanywhere-sdks
+# =============================================================================
+GITHUB_ORG = "RunanywhereAI"
+CORE_REPO = "runanywhere-sdks"
+
+# =============================================================================
+# testLocal Toggle
+# Set RA_TEST_LOCAL=1 or create .testlocal file to use local binaries
+# =============================================================================
+TEST_LOCAL = ENV['RA_TEST_LOCAL'] == '1' || File.exist?(File.join(__dir__, '.testlocal'))
+
+Pod::Spec.new do |s|
+  s.name         = "LlamaCPPBackend"
+  s.module_name  = "RunAnywhereLlama"
+  s.version      = package["version"]
+  s.summary      = package["description"]
+  s.homepage     = "https://runanywhere.com"
+  s.license      = package["license"]
+  s.authors      = "RunAnywhere AI"
+
+  s.platforms    = { :ios => "15.1" }
+  s.source       = { :git => "https://github.com/RunanywhereAI/sdks.git", :tag => "#{s.version}" }
+
+  # =============================================================================
+  # Llama Backend - RABackendLlamaCPP
+  # Downloads from runanywhere-sdks (NOT runanywhere-sdks)
+  # =============================================================================
+  if TEST_LOCAL
+    puts "[LlamaCPPBackend] Using LOCAL RABackendLlamaCPP from Frameworks/"
+    s.vendored_frameworks = "Frameworks/RABackendLLAMACPP.xcframework"
+  else
+    s.prepare_command = <<-CMD
+      set -e
+
+      FRAMEWORK_DIR="Frameworks"
+      VERSION="#{CORE_VERSION}"
+      VERSION_FILE="$FRAMEWORK_DIR/.llamacpp_version"
+
+      # Check if already downloaded with correct version
+      if [ -f "$VERSION_FILE" ] && [ -d "$FRAMEWORK_DIR/RABackendLLAMACPP.xcframework" ]; then
+        CURRENT_VERSION=$(cat "$VERSION_FILE")
+        if [ "$CURRENT_VERSION" = "$VERSION" ]; then
+          echo "✅ RABackendLLAMACPP.xcframework version $VERSION already downloaded"
+          exit 0
+        fi
+      fi
+
+      echo "📦 Downloading RABackendLlamaCPP.xcframework version $VERSION..."
+
+      mkdir -p "$FRAMEWORK_DIR"
+      rm -rf "$FRAMEWORK_DIR/RABackendLLAMACPP.xcframework"
+
+      # Download from runanywhere-sdks
+      DOWNLOAD_URL="https://github.com/#{GITHUB_ORG}/#{CORE_REPO}/releases/download/core-v$VERSION/RABackendLlamaCPP-ios-v$VERSION.zip"
+      ZIP_FILE="/tmp/RABackendLlamaCPP.zip"
+
+      echo "   URL: $DOWNLOAD_URL"
+
+      curl -L -f -o "$ZIP_FILE" "$DOWNLOAD_URL" || {
+        echo "❌ Failed to download RABackendLlamaCPP from $DOWNLOAD_URL"
+        exit 1
+      }
+
+      echo "📂 Extracting RABackendLLAMACPP.xcframework..."
+      unzip -q -o "$ZIP_FILE" -d "$FRAMEWORK_DIR/"
+      rm -f "$ZIP_FILE"
+
+      echo "$VERSION" > "$VERSION_FILE"
+
+      if [ -d "$FRAMEWORK_DIR/RABackendLLAMACPP.xcframework" ]; then
+        echo "✅ RABackendLLAMACPP.xcframework installed successfully"
+      else
+        echo "❌ RABackendLLAMACPP.xcframework extraction failed"
+        exit 1
+      fi
+    CMD
+
+    s.vendored_frameworks = "Frameworks/RABackendLLAMACPP.xcframework"
+  end
+
+  # Source files - Llama C++ implementation
+  s.source_files = [
+    "../cpp/HybridRunAnywhereLlama.cpp",
+    "../cpp/HybridRunAnywhereLlama.hpp",
+    "../cpp/bridges/**/*.{cpp,hpp}",
+  ]
+
+  # Build settings
+  s.pod_target_xcconfig = {
+    "CLANG_CXX_LANGUAGE_STANDARD" => "c++17",
+    "HEADER_SEARCH_PATHS" => [
+      "$(PODS_TARGET_SRCROOT)/../cpp",
+      "$(PODS_TARGET_SRCROOT)/../cpp/bridges",
+      "$(PODS_TARGET_SRCROOT)/Frameworks/RABackendLLAMACPP.xcframework/ios-arm64/Headers",
+      "$(PODS_TARGET_SRCROOT)/Frameworks/RABackendLLAMACPP.xcframework/ios-arm64_x86_64-simulator/Headers",
+      "$(PODS_ROOT)/Headers/Public",
+    ].join(" "),
+    "GCC_PREPROCESSOR_DEFINITIONS" => "$(inherited) HAS_LLAMACPP=1",
+    "DEFINES_MODULE" => "YES",
+    "SWIFT_OBJC_INTEROP_MODE" => "objcxx",
+  }
+
+  # Required system libraries
+  s.libraries = "c++"
+  s.frameworks = "Accelerate", "Foundation", "CoreML"
+
+  # Dependencies
+  s.dependency 'RunAnywhereCore'
+  s.dependency 'React-jsi'
+  s.dependency 'React-callinvoker'
+
+  # Load Nitrogen-generated autolinking
+  load '../nitrogen/generated/ios/RunAnywhereLlama+autolinking.rb'
+  add_nitrogen_files(s)
+
+  install_modules_dependencies(s)
+end

package/nitro.json

@@ -0,0 +1,16 @@
+{
+  "cxxNamespace": ["runanywhere", "llama"],
+  "ios": {
+    "iosModuleName": "RunAnywhereLlama"
+  },
+  "android": {
+    "androidNamespace": ["runanywhere", "llama"],
+    "androidCxxLibName": "runanywherellama"
+  },
+  "autolinking": {
+    "RunAnywhereLlama": {
+      "cpp": "HybridRunAnywhereLlama"
+    }
+  },
+  "ignorePaths": ["node_modules", "lib", "example"]
+}

package/nitrogen/generated/.gitattributes

@@ -0,0 +1 @@
+** linguist-generated=true

package/nitrogen/generated/android/kotlin/com/margelo/nitro/runanywhere/llama/runanywherellamaOnLoad.kt

@@ -0,0 +1,35 @@
+///
+/// runanywherellamaOnLoad.kt
+/// This file was generated by nitrogen. DO NOT MODIFY THIS FILE.
+/// https://github.com/mrousavy/nitro
+/// Copyright © 2026 Marc Rousavy @ Margelo
+///
+
+package com.margelo.nitro.runanywhere.llama
+
+import android.util.Log
+
+internal class runanywherellamaOnLoad {
+  companion object {
+    private const val TAG = "runanywherellamaOnLoad"
+    private var didLoad = false
+    /**
+     * Initializes the native part of "runanywherellama".
+     * This method is idempotent and can be called more than once.
+     */
+    @JvmStatic
+    fun initializeNative() {
+      if (didLoad) return
+      try {
+        Log.i(TAG, "Loading runanywherellama C++ library...")
+        System.loadLibrary("runanywherellama")
+        Log.i(TAG, "Successfully loaded runanywherellama C++ library!")
+        didLoad = true
+      } catch (e: Error) {
+        Log.e(TAG, "Failed to load runanywherellama C++ library! Is it properly installed and linked? " +
+                    "Is the name correct? (see `CMakeLists.txt`, at `add_library(...)`)", e)
+        throw e
+      }
+    }
+  }
+}