@runanywhere/core 0.17.8 → 0.18.0

package/android/src/main/include/rac/backends/rac_vlm_llamacpp.h

@@ -0,0 +1,216 @@
+/**
+ * @file rac_vlm_llamacpp.h
+ * @brief RunAnywhere Commons - LlamaCPP VLM Backend API
+ *
+ * Public C API for Vision Language Model inference using llama.cpp's
+ * multimodal (mtmd) capabilities. Supports 20+ VLM architectures including
+ * Qwen2-VL, Qwen2.5-VL, SmolVLM, LLaVA, MiniCPM-V, and more.
+ */
+
+#ifndef RAC_VLM_LLAMACPP_H
+#define RAC_VLM_LLAMACPP_H
+
+#include "rac/core/rac_error.h"
+#include "rac/core/rac_types.h"
+#include "rac/features/vlm/rac_vlm.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// =============================================================================
+// EXPORT MACRO
+// =============================================================================
+
+#if defined(RAC_LLAMACPP_BUILDING)
+#if defined(_WIN32)
+#define RAC_LLAMACPP_VLM_API __declspec(dllexport)
+#elif defined(__GNUC__) || defined(__clang__)
+#define RAC_LLAMACPP_VLM_API __attribute__((visibility("default")))
+#else
+#define RAC_LLAMACPP_VLM_API
+#endif
+#else
+#define RAC_LLAMACPP_VLM_API
+#endif
+
+// =============================================================================
+// CONFIGURATION
+// =============================================================================
+
+/**
+ * LlamaCPP VLM-specific configuration.
+ */
+typedef struct rac_vlm_llamacpp_config {
+    /** Context size (0 = auto-detect from model) */
+    int32_t context_size;
+
+    /** Number of threads for CPU inference (0 = auto-detect) */
+    int32_t num_threads;
+
+    /** Number of layers to offload to GPU (Metal on iOS/macOS, -1 = all) */
+    int32_t gpu_layers;
+
+    /** Batch size for prompt processing */
+    int32_t batch_size;
+
+    /** Number of threads for vision encoder (0 = same as num_threads) */
+    int32_t vision_threads;
+
+    /** Use GPU for vision encoding */
+    rac_bool_t use_gpu_vision;
+} rac_vlm_llamacpp_config_t;
+
+/**
+ * Default LlamaCPP VLM configuration.
+ */
+static const rac_vlm_llamacpp_config_t RAC_VLM_LLAMACPP_CONFIG_DEFAULT = {
+    .context_size = 0,      // Auto-detect
+    .num_threads = 0,       // Auto-detect
+    .gpu_layers = -1,       // All layers on GPU
+    .batch_size = 512,      //
+    .vision_threads = 0,    // Auto-detect
+    .use_gpu_vision = 1     // Use GPU for vision
+};
+
+// =============================================================================
+// LLAMACPP VLM-SPECIFIC API
+// =============================================================================
+
+/**
+ * Creates a LlamaCPP VLM service.
+ *
+ * @param model_path Path to the GGUF LLM model file
+ * @param mmproj_path Path to the mmproj vision projector GGUF file
+ * @param config LlamaCPP-specific configuration (can be NULL for defaults)
+ * @param out_handle Output: Handle to the created service
+ * @return RAC_SUCCESS or error code
+ */
+RAC_LLAMACPP_VLM_API rac_result_t rac_vlm_llamacpp_create(const char* model_path,
+                                                          const char* mmproj_path,
+                                                          const rac_vlm_llamacpp_config_t* config,
+                                                          rac_handle_t* out_handle);
+
+/**
+ * Loads a VLM model into an existing service.
+ *
+ * @param handle Service handle
+ * @param model_path Path to the GGUF LLM model file
+ * @param mmproj_path Path to the mmproj vision projector GGUF file
+ * @param config LlamaCPP configuration (can be NULL)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_LLAMACPP_VLM_API rac_result_t rac_vlm_llamacpp_load_model(
+    rac_handle_t handle, const char* model_path, const char* mmproj_path,
+    const rac_vlm_llamacpp_config_t* config);
+
+/**
+ * Unloads the current model.
+ *
+ * @param handle Service handle
+ * @return RAC_SUCCESS or error code
+ */
+RAC_LLAMACPP_VLM_API rac_result_t rac_vlm_llamacpp_unload_model(rac_handle_t handle);
+
+/**
+ * Checks if a model is loaded.
+ *
+ * @param handle Service handle
+ * @return RAC_TRUE if model is loaded, RAC_FALSE otherwise
+ */
+RAC_LLAMACPP_VLM_API rac_bool_t rac_vlm_llamacpp_is_model_loaded(rac_handle_t handle);
+
+/**
+ * Processes an image with a text prompt (blocking).
+ *
+ * @param handle Service handle
+ * @param image Image input (file path, RGB pixels, or base64)
+ * @param prompt Text prompt
+ * @param options VLM generation options (can be NULL for defaults)
+ * @param out_result Output: Generation result (caller must free text with rac_free)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_LLAMACPP_VLM_API rac_result_t rac_vlm_llamacpp_process(rac_handle_t handle,
+                                                           const rac_vlm_image_t* image,
+                                                           const char* prompt,
+                                                           const rac_vlm_options_t* options,
+                                                           rac_vlm_result_t* out_result);
+
+/**
+ * Streaming callback for VLM generation.
+ *
+ * @param token Generated token string
+ * @param is_final Whether this is the final token
+ * @param user_data User-provided context
+ * @return RAC_TRUE to continue, RAC_FALSE to stop
+ */
+typedef rac_bool_t (*rac_vlm_llamacpp_stream_callback_fn)(const char* token, rac_bool_t is_final,
+                                                          void* user_data);
+
+/**
+ * Processes an image with streaming callback.
+ *
+ * @param handle Service handle
+ * @param image Image input
+ * @param prompt Text prompt
+ * @param options VLM generation options
+ * @param callback Callback for each token
+ * @param user_data User context passed to callback
+ * @return RAC_SUCCESS or error code
+ */
+RAC_LLAMACPP_VLM_API rac_result_t rac_vlm_llamacpp_process_stream(
+    rac_handle_t handle, const rac_vlm_image_t* image, const char* prompt,
+    const rac_vlm_options_t* options, rac_vlm_llamacpp_stream_callback_fn callback, void* user_data);
+
+/**
+ * Cancels ongoing generation.
+ *
+ * @param handle Service handle
+ */
+RAC_LLAMACPP_VLM_API void rac_vlm_llamacpp_cancel(rac_handle_t handle);
+
+/**
+ * Gets model information as JSON.
+ *
+ * @param handle Service handle
+ * @param out_json Output: JSON string (caller must free with rac_free)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_LLAMACPP_VLM_API rac_result_t rac_vlm_llamacpp_get_model_info(rac_handle_t handle,
+                                                                  char** out_json);
+
+/**
+ * Destroys a LlamaCPP VLM service.
+ *
+ * @param handle Service handle to destroy
+ */
+RAC_LLAMACPP_VLM_API void rac_vlm_llamacpp_destroy(rac_handle_t handle);
+
+// =============================================================================
+// BACKEND REGISTRATION
+// =============================================================================
+
+/**
+ * Registers the LlamaCPP VLM backend with the commons module and service registries.
+ *
+ * Should be called once during SDK initialization.
+ * This registers:
+ * - Module: "llamacpp_vlm" with VISION_LANGUAGE capability
+ * - Service provider: LlamaCPP VLM provider (priority 100)
+ *
+ * @return RAC_SUCCESS or error code
+ */
+RAC_LLAMACPP_VLM_API rac_result_t rac_backend_llamacpp_vlm_register(void);
+
+/**
+ * Unregisters the LlamaCPP VLM backend.
+ *
+ * @return RAC_SUCCESS or error code
+ */
+RAC_LLAMACPP_VLM_API rac_result_t rac_backend_llamacpp_vlm_unregister(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RAC_VLM_LLAMACPP_H */

package/android/src/main/include/rac/core/capabilities/rac_lifecycle.h

@@ -45,7 +45,9 @@ typedef enum rac_resource_type {
     RAC_RESOURCE_TYPE_STT_MODEL = 1,
     RAC_RESOURCE_TYPE_TTS_VOICE = 2,
     RAC_RESOURCE_TYPE_VAD_MODEL = 3,
-    RAC_RESOURCE_TYPE_DIARIZATION_MODEL = 4
+    RAC_RESOURCE_TYPE_DIARIZATION_MODEL = 4,
+    RAC_RESOURCE_TYPE_VLM_MODEL = 5,       /**< Vision Language Model */
+    RAC_RESOURCE_TYPE_DIFFUSION_MODEL = 6  /**< Diffusion/Image Generation Model */
 } rac_resource_type_t;
 
 /**

package/android/src/main/include/rac/core/rac_core.h

@@ -324,6 +324,17 @@ RAC_API rac_result_t rac_register_model(const struct rac_model_info* model);
  */
 RAC_API rac_result_t rac_get_model(const char* model_id, struct rac_model_info** out_model);
 
+/**
+ * Gets model info from the global registry by local path.
+ * Convenience function that calls rac_model_registry_get_by_path on the global registry.
+ * Useful when loading models by path instead of model_id.
+ *
+ * @param local_path Local path to search for
+ * @param out_model Output: Model info (owned, must be freed with rac_model_info_free)
+ * @return RAC_SUCCESS on success, RAC_ERROR_NOT_FOUND if not registered
+ */
+RAC_API rac_result_t rac_get_model_by_path(const char* local_path, struct rac_model_info** out_model);
+
 #ifdef __cplusplus
 }
 #endif

package/android/src/main/include/rac/core/rac_types.h

@@ -166,12 +166,14 @@ typedef struct rac_memory_info {
  */
 typedef enum rac_capability {
     RAC_CAPABILITY_UNKNOWN = 0,
-    RAC_CAPABILITY_TEXT_GENERATION = 1, /**< LLM text generation */
-    RAC_CAPABILITY_EMBEDDINGS = 2,      /**< Text embeddings */
-    RAC_CAPABILITY_STT = 3,             /**< Speech-to-text */
-    RAC_CAPABILITY_TTS = 4,             /**< Text-to-speech */
-    RAC_CAPABILITY_VAD = 5,             /**< Voice activity detection */
-    RAC_CAPABILITY_DIARIZATION = 6,     /**< Speaker diarization */
+    RAC_CAPABILITY_TEXT_GENERATION = 1,   /**< LLM text generation */
+    RAC_CAPABILITY_EMBEDDINGS = 2,        /**< Text embeddings */
+    RAC_CAPABILITY_STT = 3,               /**< Speech-to-text */
+    RAC_CAPABILITY_TTS = 4,               /**< Text-to-speech */
+    RAC_CAPABILITY_VAD = 5,               /**< Voice activity detection */
+    RAC_CAPABILITY_DIARIZATION = 6,       /**< Speaker diarization */
+    RAC_CAPABILITY_VISION_LANGUAGE = 7,   /**< Vision-language model (VLM) */
+    RAC_CAPABILITY_DIFFUSION = 8,         /**< Image generation (Stable Diffusion) */
 } rac_capability_t;
 
 /**

package/android/src/main/include/rac/features/diffusion/rac_diffusion.h

@@ -0,0 +1,22 @@
+/**
+ * @file rac_diffusion.h
+ * @brief RunAnywhere Commons - Diffusion Feature Umbrella Header
+ *
+ * Include this header to use diffusion (image generation) capabilities.
+ * This provides text-to-image, image-to-image, and inpainting using
+ * Stable Diffusion models.
+ *
+ * Supported backends:
+ * - CoreML (Apple platforms) - via Platform backend
+ * - ONNX Runtime (cross-platform) - via ONNX backend
+ */
+
+#ifndef RAC_DIFFUSION_H
+#define RAC_DIFFUSION_H
+
+#include "rac/features/diffusion/rac_diffusion_component.h"
+#include "rac/features/diffusion/rac_diffusion_service.h"
+#include "rac/features/diffusion/rac_diffusion_tokenizer.h"
+#include "rac/features/diffusion/rac_diffusion_types.h"
+
+#endif /* RAC_DIFFUSION_H */

package/android/src/main/include/rac/features/diffusion/rac_diffusion_component.h

@@ -0,0 +1,263 @@
+/**
+ * @file rac_diffusion_component.h
+ * @brief RunAnywhere Commons - Diffusion Capability Component
+ *
+ * Actor-based diffusion capability that owns model lifecycle and generation.
+ * Uses lifecycle manager for unified lifecycle + analytics handling.
+ *
+ * Supports:
+ * - Text-to-image generation
+ * - Image-to-image transformation
+ * - Inpainting with mask
+ * - Progress reporting with optional intermediate images
+ */
+
+#ifndef RAC_DIFFUSION_COMPONENT_H
+#define RAC_DIFFUSION_COMPONENT_H
+
+#include "rac/core/capabilities/rac_lifecycle.h"
+#include "rac/core/rac_error.h"
+#include "rac/features/diffusion/rac_diffusion_types.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// =============================================================================
+// DIFFUSION COMPONENT API - Component lifecycle and generation
+// =============================================================================
+
+/**
+ * @brief Create a diffusion capability component
+ *
+ * @param out_handle Output: Handle to the component
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_diffusion_component_create(rac_handle_t* out_handle);
+
+/**
+ * @brief Configure the diffusion component
+ *
+ * @param handle Component handle
+ * @param config Configuration
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_diffusion_component_configure(rac_handle_t handle,
+                                                       const rac_diffusion_config_t* config);
+
+/**
+ * @brief Check if model is loaded
+ *
+ * @param handle Component handle
+ * @return RAC_TRUE if loaded, RAC_FALSE otherwise
+ */
+RAC_API rac_bool_t rac_diffusion_component_is_loaded(rac_handle_t handle);
+
+/**
+ * @brief Get current model ID
+ *
+ * @param handle Component handle
+ * @return Current model ID (NULL if not loaded)
+ */
+RAC_API const char* rac_diffusion_component_get_model_id(rac_handle_t handle);
+
+/**
+ * @brief Load a diffusion model
+ *
+ * @param handle Component handle
+ * @param model_path Path to the model directory
+ * @param model_id Model identifier for telemetry
+ * @param model_name Human-readable model name
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_diffusion_component_load_model(rac_handle_t handle, const char* model_path,
+                                                        const char* model_id,
+                                                        const char* model_name);
+
+/**
+ * @brief Unload the current model
+ *
+ * @param handle Component handle
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_diffusion_component_unload(rac_handle_t handle);
+
+/**
+ * @brief Cleanup and reset the component
+ *
+ * @param handle Component handle
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_diffusion_component_cleanup(rac_handle_t handle);
+
+/**
+ * @brief Cancel ongoing generation
+ *
+ * Best-effort cancellation.
+ *
+ * @param handle Component handle
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_diffusion_component_cancel(rac_handle_t handle);
+
+/**
+ * @brief Generate an image (non-streaming)
+ *
+ * Blocking call that generates an image from the prompt.
+ *
+ * @param handle Component handle
+ * @param options Generation options
+ * @param out_result Output: Generation result
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_diffusion_component_generate(rac_handle_t handle,
+                                                      const rac_diffusion_options_t* options,
+                                                      rac_diffusion_result_t* out_result);
+
+/**
+ * @brief Generate an image with progress callbacks
+ *
+ * Non-blocking call with progress reporting via callbacks.
+ *
+ * @param handle Component handle
+ * @param options Generation options
+ * @param progress_callback Called for each progress update
+ * @param complete_callback Called when generation completes
+ * @param error_callback Called on error
+ * @param user_data User context passed to callbacks
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_diffusion_component_generate_with_callbacks(
+    rac_handle_t handle, const rac_diffusion_options_t* options,
+    rac_diffusion_progress_callback_fn progress_callback,
+    rac_diffusion_complete_callback_fn complete_callback,
+    rac_diffusion_error_callback_fn error_callback, void* user_data);
+
+// =============================================================================
+// JSON CONVENIENCE HELPERS
+// =============================================================================
+
+/**
+ * @brief Configure diffusion component from JSON
+ *
+ * JSON schema (flat object):
+ * {
+ *   "model_id": "optional-model-id",
+ *   "model_variant": 0 | "sd15" | "sd21" | "sdxl" | "sdxl_turbo" | "sdxs" | "lcm",
+ *   "enable_safety_checker": true/false,
+ *   "reduce_memory": true/false,
+ *   "tokenizer_source": 0 | 1 | 2 | 99,
+ *   "tokenizer_custom_url": "https://..."
+ * }
+ *
+ * @param handle Component handle
+ * @param config_json JSON string
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_diffusion_component_configure_json(rac_handle_t handle,
+                                                            const char* config_json);
+
+/**
+ * @brief Generate image from JSON options
+ *
+ * JSON schema (flat object):
+ * {
+ *   "prompt": "text prompt",
+ *   "negative_prompt": "optional",
+ *   "width": 512,
+ *   "height": 512,
+ *   "steps": 28,
+ *   "guidance_scale": 7.5,
+ *   "seed": -1,
+ *   "scheduler": 0 | "dpm++_2m_karras" | "dpm++_2m" | "dpm++_2m_sde" | "ddim" | "euler" | "euler_a" | "pndm" | "lms",
+ *   "mode": 0 | "txt2img" | "img2img" | "inpainting",
+ *   "denoise_strength": 0.75,
+ *   "report_intermediate_images": false,
+ *   "progress_stride": 1
+ * }
+ *
+ * @param handle Component handle
+ * @param options_json JSON string
+ * @param input_image_data Optional input image bytes (PNG/JPEG or RGBA)
+ * @param input_image_size Size of input image bytes
+ * @param mask_data Optional mask image bytes (PNG/JPEG or grayscale)
+ * @param mask_size Size of mask bytes
+ * @param out_json Output JSON (caller must free with rac_free)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_diffusion_component_generate_json(
+    rac_handle_t handle, const char* options_json, const uint8_t* input_image_data,
+    size_t input_image_size, const uint8_t* mask_data, size_t mask_size, char** out_json);
+
+/**
+ * @brief Get diffusion info as JSON
+ *
+ * Output schema:
+ * {
+ *   "is_ready": true/false,
+ *   "current_model": "id",
+ *   "model_variant": 0,
+ *   "supports_text_to_image": true/false,
+ *   "supports_image_to_image": true/false,
+ *   "supports_inpainting": true/false,
+ *   "safety_checker_enabled": true/false,
+ *   "max_width": 512,
+ *   "max_height": 512
+ * }
+ *
+ * @param handle Component handle
+ * @param out_json Output JSON (caller must free with rac_free)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_diffusion_component_get_info_json(rac_handle_t handle, char** out_json);
+
+/**
+ * @brief Get supported capabilities
+ *
+ * Returns a bitmask of supported capabilities.
+ *
+ * @param handle Component handle
+ * @return Capability bitmask (RAC_DIFFUSION_CAP_* flags)
+ */
+RAC_API uint32_t rac_diffusion_component_get_capabilities(rac_handle_t handle);
+
+/**
+ * @brief Get service information
+ *
+ * @param handle Component handle
+ * @param out_info Output: Service information
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_diffusion_component_get_info(rac_handle_t handle,
+                                                      rac_diffusion_info_t* out_info);
+
+/**
+ * @brief Get lifecycle state
+ *
+ * @param handle Component handle
+ * @return Current lifecycle state
+ */
+RAC_API rac_lifecycle_state_t rac_diffusion_component_get_state(rac_handle_t handle);
+
+/**
+ * @brief Get lifecycle metrics
+ *
+ * @param handle Component handle
+ * @param out_metrics Output: Lifecycle metrics
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_diffusion_component_get_metrics(rac_handle_t handle,
+                                                         rac_lifecycle_metrics_t* out_metrics);
+
+/**
+ * @brief Destroy the diffusion component
+ *
+ * @param handle Component handle
+ */
+RAC_API void rac_diffusion_component_destroy(rac_handle_t handle);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RAC_DIFFUSION_COMPONENT_H */