@runanywhere/llamacpp 0.16.0

package/cpp/HybridRunAnywhereLlama.cpp

@@ -0,0 +1,346 @@
+/**
+ * HybridRunAnywhereLlama.cpp
+ *
+ * Nitrogen HybridObject implementation for RunAnywhere Llama backend.
+ *
+ * Llama-specific implementation for text generation using LlamaCPP.
+ *
+ * NOTE: LlamaCPP backend is REQUIRED and always linked via the build system.
+ */
+
+#include "HybridRunAnywhereLlama.hpp"
+
+// Llama bridges
+#include "bridges/LLMBridge.hpp"
+#include "bridges/StructuredOutputBridge.hpp"
+
+// Backend registration header - always available
+extern "C" {
+#include "rac_llm_llamacpp.h"
+}
+
+// Unified logging via rac_logger.h
+#include "rac_logger.h"
+
+#include <sstream>
+#include <chrono>
+#include <vector>
+#include <stdexcept>
+
+// Log category for this module
+#define LOG_CATEGORY "LLM.LlamaCpp"
+
+namespace margelo::nitro::runanywhere::llama {
+
+using namespace ::runanywhere::bridges;
+
+// ============================================================================
+// JSON Utilities
+// ============================================================================
+
+namespace {
+
+int extractIntValue(const std::string& json, const std::string& key, int defaultValue) {
+  std::string searchKey = "\"" + key + "\":";
+  size_t pos = json.find(searchKey);
+  if (pos == std::string::npos) return defaultValue;
+  pos += searchKey.length();
+  while (pos < json.size() && (json[pos] == ' ' || json[pos] == '\t')) pos++;
+  if (pos >= json.size()) return defaultValue;
+  return std::stoi(json.substr(pos));
+}
+
+float extractFloatValue(const std::string& json, const std::string& key, float defaultValue) {
+  std::string searchKey = "\"" + key + "\":";
+  size_t pos = json.find(searchKey);
+  if (pos == std::string::npos) return defaultValue;
+  pos += searchKey.length();
+  while (pos < json.size() && (json[pos] == ' ' || json[pos] == '\t')) pos++;
+  if (pos >= json.size()) return defaultValue;
+  return std::stof(json.substr(pos));
+}
+
+std::string extractStringValue(const std::string& json, const std::string& key, const std::string& defaultValue = "") {
+  std::string searchKey = "\"" + key + "\":\"";
+  size_t pos = json.find(searchKey);
+  if (pos == std::string::npos) return defaultValue;
+  pos += searchKey.length();
+  size_t endPos = json.find("\"", pos);
+  if (endPos == std::string::npos) return defaultValue;
+  return json.substr(pos, endPos - pos);
+}
+
+std::string buildJsonObject(const std::vector<std::pair<std::string, std::string>>& keyValues) {
+  std::string result = "{";
+  for (size_t i = 0; i < keyValues.size(); i++) {
+    if (i > 0) result += ",";
+    result += "\"" + keyValues[i].first + "\":" + keyValues[i].second;
+  }
+  result += "}";
+  return result;
+}
+
+std::string jsonString(const std::string& value) {
+  std::string escaped = "\"";
+  for (char c : value) {
+    if (c == '"') escaped += "\\\"";
+    else if (c == '\\') escaped += "\\\\";
+    else if (c == '\n') escaped += "\\n";
+    else if (c == '\r') escaped += "\\r";
+    else if (c == '\t') escaped += "\\t";
+    else escaped += c;
+  }
+  escaped += "\"";
+  return escaped;
+}
+
+} // anonymous namespace
+
+// ============================================================================
+// Constructor / Destructor
+// ============================================================================
+
+HybridRunAnywhereLlama::HybridRunAnywhereLlama() : HybridObject(TAG) {
+  RAC_LOG_DEBUG(LOG_CATEGORY, "HybridRunAnywhereLlama constructor - Llama backend module");
+}
+
+HybridRunAnywhereLlama::~HybridRunAnywhereLlama() {
+  RAC_LOG_DEBUG(LOG_CATEGORY, "HybridRunAnywhereLlama destructor");
+  LLMBridge::shared().destroy();
+}
+
+// ============================================================================
+// Backend Registration
+// ============================================================================
+
+std::shared_ptr<Promise<bool>> HybridRunAnywhereLlama::registerBackend() {
+  return Promise<bool>::async([this]() {
+    RAC_LOG_DEBUG(LOG_CATEGORY, "Registering LlamaCPP backend with C++ registry");
+
+    rac_result_t result = rac_backend_llamacpp_register();
+    // RAC_SUCCESS (0) or RAC_ERROR_MODULE_ALREADY_REGISTERED (-4) are both OK
+    if (result == RAC_SUCCESS || result == -4) {
+      RAC_LOG_INFO(LOG_CATEGORY, "LlamaCPP backend registered successfully");
+      isRegistered_ = true;
+      return true;
+    } else {
+      RAC_LOG_ERROR(LOG_CATEGORY, "LlamaCPP registration failed with code: %d", result);
+      setLastError("LlamaCPP registration failed with error: " + std::to_string(result));
+      throw std::runtime_error("LlamaCPP registration failed with error: " + std::to_string(result));
+    }
+  });
+}
+
+std::shared_ptr<Promise<bool>> HybridRunAnywhereLlama::unregisterBackend() {
+  return Promise<bool>::async([this]() {
+    RAC_LOG_DEBUG(LOG_CATEGORY, "Unregistering LlamaCPP backend");
+
+    rac_result_t result = rac_backend_llamacpp_unregister();
+    isRegistered_ = false;
+    if (result != RAC_SUCCESS) {
+      RAC_LOG_ERROR(LOG_CATEGORY, "LlamaCPP unregistration failed with code: %d", result);
+      throw std::runtime_error("LlamaCPP unregistration failed with error: " + std::to_string(result));
+    }
+    return true;
+  });
+}
+
+std::shared_ptr<Promise<bool>> HybridRunAnywhereLlama::isBackendRegistered() {
+  return Promise<bool>::async([this]() {
+    return isRegistered_;
+  });
+}
+
+// ============================================================================
+// Model Loading
+// ============================================================================
+
+std::shared_ptr<Promise<bool>> HybridRunAnywhereLlama::loadModel(
+    const std::string& path,
+    const std::optional<std::string>& modelId,
+    const std::optional<std::string>& modelName,
+    const std::optional<std::string>& configJson) {
+  return Promise<bool>::async([this, path, modelId, modelName, configJson]() {
+    std::lock_guard<std::mutex> lock(modelMutex_);
+
+    RAC_LOG_INFO(LOG_CATEGORY, "Loading Llama model: %s", path.c_str());
+
+    std::string id = modelId.value_or("");
+    std::string name = modelName.value_or("");
+
+    // Call with correct 4-arg signature (path, modelId, modelName)
+    // LLMBridge::loadModel will throw on error
+    auto result = LLMBridge::shared().loadModel(path, id, name);
+    if (result != 0) {
+      std::string error = "Failed to load Llama model: " + path + " (error: " + std::to_string(result) + ")";
+      setLastError(error);
+      throw std::runtime_error(error);
+    }
+    return true;
+  });
+}
+
+std::shared_ptr<Promise<bool>> HybridRunAnywhereLlama::isModelLoaded() {
+  return Promise<bool>::async([]() {
+    return LLMBridge::shared().isLoaded();
+  });
+}
+
+std::shared_ptr<Promise<bool>> HybridRunAnywhereLlama::unloadModel() {
+  return Promise<bool>::async([this]() {
+    std::lock_guard<std::mutex> lock(modelMutex_);
+    auto result = LLMBridge::shared().unload();
+    return result == 0;
+  });
+}
+
+std::shared_ptr<Promise<std::string>> HybridRunAnywhereLlama::getModelInfo() {
+  return Promise<std::string>::async([]() {
+    if (!LLMBridge::shared().isLoaded()) {
+      return std::string("{}");
+    }
+    return buildJsonObject({
+      {"loaded", "true"},
+      {"backend", jsonString("llamacpp")}
+    });
+  });
+}
+
+// ============================================================================
+// Text Generation
+// ============================================================================
+
+std::shared_ptr<Promise<std::string>> HybridRunAnywhereLlama::generate(
+    const std::string& prompt,
+    const std::optional<std::string>& optionsJson) {
+  return Promise<std::string>::async([this, prompt, optionsJson]() {
+    if (!LLMBridge::shared().isLoaded()) {
+      setLastError("Model not loaded");
+      throw std::runtime_error("LLMBridge: Model not loaded. Call loadModel() first.");
+    }
+
+    LLMOptions options;
+    if (optionsJson.has_value()) {
+      options.maxTokens = extractIntValue(*optionsJson, "max_tokens", 512);
+      options.temperature = extractFloatValue(*optionsJson, "temperature", 0.7f);
+      options.topP = extractFloatValue(*optionsJson, "top_p", 0.9f);
+      options.topK = extractIntValue(*optionsJson, "top_k", 40);
+    }
+
+    RAC_LOG_DEBUG(LOG_CATEGORY, "Generating with prompt: %.50s...", prompt.c_str());
+
+    auto startTime = std::chrono::high_resolution_clock::now();
+    // LLMBridge::generate will throw on error
+    auto result = LLMBridge::shared().generate(prompt, options);
+    auto endTime = std::chrono::high_resolution_clock::now();
+    auto durationMs = std::chrono::duration_cast<std::chrono::milliseconds>(
+        endTime - startTime).count();
+
+    return buildJsonObject({
+      {"text", jsonString(result.text)},
+      {"tokensUsed", std::to_string(result.tokenCount)},
+      {"latencyMs", std::to_string(durationMs)},
+      {"cancelled", result.cancelled ? "true" : "false"}
+    });
+  });
+}
+
+std::shared_ptr<Promise<std::string>> HybridRunAnywhereLlama::generateStream(
+    const std::string& prompt,
+    const std::string& optionsJson,
+    const std::function<void(const std::string&, bool)>& callback) {
+  return Promise<std::string>::async([this, prompt, optionsJson, callback]() {
+    if (!LLMBridge::shared().isLoaded()) {
+      setLastError("Model not loaded");
+      throw std::runtime_error("LLMBridge: Model not loaded. Call loadModel() first.");
+    }
+
+    LLMOptions options;
+    options.maxTokens = extractIntValue(optionsJson, "max_tokens", 512);
+    options.temperature = extractFloatValue(optionsJson, "temperature", 0.7f);
+
+    std::string fullResponse;
+    std::string streamError;
+
+    LLMStreamCallbacks streamCallbacks;
+    streamCallbacks.onToken = [&callback, &fullResponse](const std::string& token) -> bool {
+      fullResponse += token;
+      if (callback) {
+        callback(token, false);
+      }
+      return true;
+    };
+    streamCallbacks.onComplete = [&callback](const std::string&, int, double) {
+      if (callback) {
+        callback("", true);
+      }
+    };
+    streamCallbacks.onError = [this, &streamError](int code, const std::string& message) {
+      setLastError(message);
+      streamError = message;
+    };
+
+    LLMBridge::shared().generateStream(prompt, options, streamCallbacks);
+
+    if (!streamError.empty()) {
+      throw std::runtime_error("LLMBridge: Stream generation failed: " + streamError);
+    }
+
+    return fullResponse;
+  });
+}
+
+std::shared_ptr<Promise<bool>> HybridRunAnywhereLlama::cancelGeneration() {
+  return Promise<bool>::async([]() {
+    LLMBridge::shared().cancel();
+    return true;
+  });
+}
+
+// ============================================================================
+// Structured Output
+// ============================================================================
+
+std::shared_ptr<Promise<std::string>> HybridRunAnywhereLlama::generateStructured(
+    const std::string& prompt,
+    const std::string& schema,
+    const std::optional<std::string>& optionsJson) {
+  return Promise<std::string>::async([this, prompt, schema, optionsJson]() {
+    auto result = StructuredOutputBridge::shared().generate(
+      prompt, schema, optionsJson.value_or("")
+    );
+
+    if (result.success) {
+      return result.json;
+    } else {
+      setLastError(result.error);
+      return buildJsonObject({{"error", jsonString(result.error)}});
+    }
+  });
+}
+
+// ============================================================================
+// Utilities
+// ============================================================================
+
+std::shared_ptr<Promise<std::string>> HybridRunAnywhereLlama::getLastError() {
+  return Promise<std::string>::async([this]() { return lastError_; });
+}
+
+std::shared_ptr<Promise<double>> HybridRunAnywhereLlama::getMemoryUsage() {
+  return Promise<double>::async([]() {
+    // TODO: Get memory usage from LlamaCPP
+    return 0.0;
+  });
+}
+
+// ============================================================================
+// Helper Methods
+// ============================================================================
+
+void HybridRunAnywhereLlama::setLastError(const std::string& error) {
+  lastError_ = error;
+  RAC_LOG_ERROR(LOG_CATEGORY, "Error: %s", error.c_str());
+}
+
+} // namespace margelo::nitro::runanywhere::llama

package/cpp/HybridRunAnywhereLlama.hpp

@@ -0,0 +1,107 @@
+/**
+ * HybridRunAnywhereLlama.hpp
+ *
+ * Nitrogen HybridObject implementation for RunAnywhere Llama backend.
+ * This single C++ file works on both iOS and Android.
+ *
+ * Llama-specific implementation:
+ * - Backend Registration
+ * - Model Loading/Unloading
+ * - Text Generation (streaming and non-streaming)
+ * - Structured Output (JSON schema generation)
+ *
+ * Matches Swift SDK: LlamaCPPRuntime/LlamaCPP.swift
+ *
+ * The HybridRunAnywhereLlamaSpec base class is auto-generated by Nitrogen
+ * from src/specs/RunAnywhereLlama.nitro.ts
+ */
+
+#pragma once
+
+// Include the generated spec header (created by nitrogen)
+#if __has_include(<NitroModules/HybridObject.hpp>)
+#include "HybridRunAnywhereLlamaSpec.hpp"
+#else
+// Fallback include path during development
+#include "../nitrogen/generated/shared/c++/HybridRunAnywhereLlamaSpec.hpp"
+#endif
+
+#include <mutex>
+#include <string>
+
+namespace margelo::nitro::runanywhere::llama {
+
+/**
+ * HybridRunAnywhereLlama - Llama backend native implementation
+ *
+ * Implements the RunAnywhereLlama interface defined in RunAnywhereLlama.nitro.ts
+ * Delegates to LLMBridge and StructuredOutputBridge for actual inference.
+ */
+class HybridRunAnywhereLlama : public HybridRunAnywhereLlamaSpec {
+public:
+  HybridRunAnywhereLlama();
+  ~HybridRunAnywhereLlama();
+
+  // ============================================================================
+  // Backend Registration
+  // ============================================================================
+
+  std::shared_ptr<Promise<bool>> registerBackend() override;
+  std::shared_ptr<Promise<bool>> unregisterBackend() override;
+  std::shared_ptr<Promise<bool>> isBackendRegistered() override;
+
+  // ============================================================================
+  // Model Loading
+  // ============================================================================
+
+  std::shared_ptr<Promise<bool>> loadModel(
+    const std::string& path,
+    const std::optional<std::string>& modelId,
+    const std::optional<std::string>& modelName,
+    const std::optional<std::string>& configJson) override;
+  std::shared_ptr<Promise<bool>> isModelLoaded() override;
+  std::shared_ptr<Promise<bool>> unloadModel() override;
+  std::shared_ptr<Promise<std::string>> getModelInfo() override;
+
+  // ============================================================================
+  // Text Generation
+  // ============================================================================
+
+  std::shared_ptr<Promise<std::string>> generate(
+    const std::string& prompt,
+    const std::optional<std::string>& optionsJson) override;
+  std::shared_ptr<Promise<std::string>> generateStream(
+    const std::string& prompt,
+    const std::string& optionsJson,
+    const std::function<void(const std::string&, bool)>& callback) override;
+  std::shared_ptr<Promise<bool>> cancelGeneration() override;
+
+  // ============================================================================
+  // Structured Output
+  // ============================================================================
+
+  std::shared_ptr<Promise<std::string>> generateStructured(
+    const std::string& prompt,
+    const std::string& schema,
+    const std::optional<std::string>& optionsJson) override;
+
+  // ============================================================================
+  // Utilities
+  // ============================================================================
+
+  std::shared_ptr<Promise<std::string>> getLastError() override;
+  std::shared_ptr<Promise<double>> getMemoryUsage() override;
+
+private:
+  // Thread safety
+  std::mutex modelMutex_;
+
+  // State tracking
+  std::string lastError_;
+  bool isRegistered_ = false;
+
+  // Helper methods
+  void setLastError(const std::string& error);
+};
+
+} // namespace margelo::nitro::runanywhere::llama

package/cpp/bridges/LLMBridge.cpp

@@ -0,0 +1,209 @@
+/**
+ * @file LLMBridge.cpp
+ * @brief LLM capability bridge implementation
+ *
+ * NOTE: RACommons and LlamaCPP backend are REQUIRED and always linked via the build system.
+ */
+
+#include "LLMBridge.hpp"
+#include <stdexcept>
+
+namespace runanywhere {
+namespace bridges {
+
+LLMBridge& LLMBridge::shared() {
+    static LLMBridge instance;
+    return instance;
+}
+
+LLMBridge::LLMBridge() = default;
+
+LLMBridge::~LLMBridge() {
+    destroy();
+}
+
+bool LLMBridge::isLoaded() const {
+    if (handle_) {
+        return rac_llm_component_is_loaded(handle_) == RAC_TRUE;
+    }
+    return false;
+}
+
+std::string LLMBridge::currentModelId() const {
+    return loadedModelId_;
+}
+
+rac_result_t LLMBridge::loadModel(const std::string& modelPath,
+                                  const std::string& modelId,
+                                  const std::string& modelName) {
+    // Create component if needed
+    if (!handle_) {
+        rac_result_t result = rac_llm_component_create(&handle_);
+        if (result != RAC_SUCCESS) {
+            throw std::runtime_error("LLMBridge: Failed to create LLM component. Error: " + std::to_string(result));
+        }
+    }
+
+    // Use modelPath as modelId if not provided
+    std::string effectiveModelId = modelId.empty() ? modelPath : modelId;
+    std::string effectiveModelName = modelName.empty() ? effectiveModelId : modelName;
+
+    // Unload existing model if different
+    if (isLoaded() && loadedModelId_ != effectiveModelId) {
+        rac_llm_component_unload(handle_);
+    }
+
+    // Load new model with correct 4-arg signature
+    // rac_llm_component_load_model(handle, model_path, model_id, model_name)
+    rac_result_t result = rac_llm_component_load_model(
+        handle_,
+        modelPath.c_str(),
+        effectiveModelId.c_str(),
+        effectiveModelName.c_str()
+    );
+    if (result == RAC_SUCCESS) {
+        loadedModelId_ = effectiveModelId;
+    } else {
+        throw std::runtime_error("LLMBridge: Failed to load LLM model '" + effectiveModelId + "'. Error: " + std::to_string(result));
+    }
+    return result;
+}
+
+rac_result_t LLMBridge::unload() {
+    if (handle_) {
+        rac_result_t result = rac_llm_component_unload(handle_);
+        if (result == RAC_SUCCESS) {
+            loadedModelId_.clear();
+        } else {
+            throw std::runtime_error("LLMBridge: Failed to unload LLM model. Error: " + std::to_string(result));
+        }
+        return result;
+    }
+    loadedModelId_.clear();
+    return RAC_SUCCESS;
+}
+
+void LLMBridge::cleanup() {
+    if (handle_) {
+        rac_llm_component_cleanup(handle_);
+    }
+    loadedModelId_.clear();
+}
+
+void LLMBridge::cancel() {
+    cancellationRequested_ = true;
+    if (handle_) {
+        rac_llm_component_cancel(handle_);
+    }
+}
+
+void LLMBridge::destroy() {
+    if (handle_) {
+        rac_llm_component_destroy(handle_);
+        handle_ = nullptr;
+    }
+    loadedModelId_.clear();
+}
+
+LLMResult LLMBridge::generate(const std::string& prompt, const LLMOptions& options) {
+    LLMResult result;
+    cancellationRequested_ = false;
+
+    if (!handle_ || !isLoaded()) {
+        throw std::runtime_error("LLMBridge: LLM model not loaded. Call loadModel() first.");
+    }
+
+    rac_llm_options_t racOptions = {};
+    racOptions.max_tokens = options.maxTokens;
+    racOptions.temperature = static_cast<float>(options.temperature);
+    racOptions.top_p = static_cast<float>(options.topP);
+    // NOTE: top_k is not available in rac_llm_options_t, only top_p
+
+    rac_llm_result_t racResult = {};
+    rac_result_t status = rac_llm_component_generate(handle_, prompt.c_str(),
+                                                      &racOptions, &racResult);
+
+    if (status == RAC_SUCCESS) {
+        if (racResult.text) {
+            result.text = racResult.text;
+        }
+        result.tokenCount = racResult.completion_tokens;
+        result.durationMs = static_cast<double>(racResult.total_time_ms);
+    } else {
+        throw std::runtime_error("LLMBridge: Text generation failed with error code: " + std::to_string(status));
+    }
+
+    result.cancelled = cancellationRequested_;
+    return result;
+}
+
+void LLMBridge::generateStream(const std::string& prompt, const LLMOptions& options,
+                               const LLMStreamCallbacks& callbacks) {
+    cancellationRequested_ = false;
+
+    if (!handle_ || !isLoaded()) {
+        if (callbacks.onError) {
+            callbacks.onError(-4, "LLM model not loaded. Call loadModel() first.");
+        }
+        return;
+    }
+
+    rac_llm_options_t racOptions = {};
+    racOptions.max_tokens = options.maxTokens;
+    racOptions.temperature = static_cast<float>(options.temperature);
+    racOptions.top_p = static_cast<float>(options.topP);
+    // NOTE: top_k is not available in rac_llm_options_t, only top_p
+
+    // Stream context for callbacks
+    struct StreamContext {
+        const LLMStreamCallbacks* callbacks;
+        bool* cancellationRequested;
+        std::string accumulatedText;
+    };
+
+    StreamContext ctx = { &callbacks, &cancellationRequested_, "" };
+
+    auto tokenCallback = [](const char* token, void* user_data) -> rac_bool_t {
+        auto* ctx = static_cast<StreamContext*>(user_data);
+        if (*ctx->cancellationRequested) {
+            return RAC_FALSE;
+        }
+        if (ctx->callbacks->onToken && token) {
+            ctx->accumulatedText += token;
+            return ctx->callbacks->onToken(token) ? RAC_TRUE : RAC_FALSE;
+        }
+        return RAC_TRUE;
+    };
+
+    auto completeCallback = [](const rac_llm_result_t* result, void* user_data) {
+        auto* ctx = static_cast<StreamContext*>(user_data);
+        if (ctx->callbacks->onComplete) {
+            ctx->callbacks->onComplete(
+                ctx->accumulatedText,
+                result ? result->completion_tokens : 0,
+                result ? static_cast<double>(result->total_time_ms) : 0.0
+            );
+        }
+    };
+
+    auto errorCallback = [](rac_result_t error_code, const char* error_message,
+                           void* user_data) {
+        auto* ctx = static_cast<StreamContext*>(user_data);
+        if (ctx->callbacks->onError) {
+            ctx->callbacks->onError(error_code, error_message ? error_message : "Unknown error");
+        }
+    };
+
+    rac_llm_component_generate_stream(handle_, prompt.c_str(), &racOptions,
+                                      tokenCallback, completeCallback, errorCallback, &ctx);
+}
+
+rac_lifecycle_state_t LLMBridge::getState() const {
+    if (handle_) {
+        return rac_llm_component_get_state(handle_);
+    }
+    return RAC_LIFECYCLE_STATE_IDLE;
+}
+
+} // namespace bridges
+} // namespace runanywhere