@runanywhere/core 0.17.7 → 0.18.0

package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_vlm_component.h

@@ -0,0 +1,168 @@
+/**
+ * @file rac_vlm_component.h
+ * @brief RunAnywhere Commons - VLM Capability Component
+ *
+ * Actor-based VLM capability that owns model lifecycle and generation.
+ * Uses lifecycle manager for unified lifecycle + analytics handling.
+ */
+
+#ifndef RAC_VLM_COMPONENT_H
+#define RAC_VLM_COMPONENT_H
+
+#include <RACommons/rac_lifecycle.h>
+#include <RACommons/rac_error.h>
+#include <RACommons/rac_vlm_types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// =============================================================================
+// VLM COMPONENT API
+// =============================================================================
+
+/**
+ * @brief Create a VLM capability component
+ *
+ * @param out_handle Output: Handle to the component
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_create(rac_handle_t* out_handle);
+
+/**
+ * @brief Configure the VLM component
+ *
+ * @param handle Component handle
+ * @param config Configuration
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_configure(rac_handle_t handle,
+                                                 const rac_vlm_config_t* config);
+
+/**
+ * @brief Check if model is loaded
+ *
+ * @param handle Component handle
+ * @return RAC_TRUE if loaded, RAC_FALSE otherwise
+ */
+RAC_API rac_bool_t rac_vlm_component_is_loaded(rac_handle_t handle);
+
+/**
+ * @brief Get current model ID
+ *
+ * @param handle Component handle
+ * @return Current model ID (NULL if not loaded)
+ */
+RAC_API const char* rac_vlm_component_get_model_id(rac_handle_t handle);
+
+/**
+ * @brief Load a VLM model
+ *
+ * @param handle Component handle
+ * @param model_path File path to the main model (LLM weights) - REQUIRED
+ * @param mmproj_path File path to the vision projector (required for llama.cpp, NULL for MLX)
+ * @param model_id Model identifier for telemetry (optional: if NULL, defaults to model_path)
+ * @param model_name Human-readable model name (optional: if NULL, defaults to model_id)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_load_model(rac_handle_t handle, const char* model_path,
+                                                  const char* mmproj_path, const char* model_id,
+                                                  const char* model_name);
+
+/**
+ * @brief Unload the current model
+ *
+ * @param handle Component handle
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_unload(rac_handle_t handle);
+
+/**
+ * @brief Cleanup and reset the component
+ *
+ * @param handle Component handle
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_cleanup(rac_handle_t handle);
+
+/**
+ * @brief Cancel ongoing generation
+ *
+ * Best-effort cancellation.
+ *
+ * @param handle Component handle
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_cancel(rac_handle_t handle);
+
+/**
+ * @brief Process an image with text prompt (non-streaming)
+ *
+ * @param handle Component handle
+ * @param image Image input
+ * @param prompt Text prompt
+ * @param options Generation options (can be NULL for defaults)
+ * @param out_result Output: Generation result
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_process(rac_handle_t handle, const rac_vlm_image_t* image,
+                                               const char* prompt, const rac_vlm_options_t* options,
+                                               rac_vlm_result_t* out_result);
+
+/**
+ * @brief Check if streaming is supported
+ *
+ * @param handle Component handle
+ * @return RAC_TRUE if streaming supported, RAC_FALSE otherwise
+ */
+RAC_API rac_bool_t rac_vlm_component_supports_streaming(rac_handle_t handle);
+
+/**
+ * @brief Process an image with streaming
+ *
+ * @param handle Component handle
+ * @param image Image input
+ * @param prompt Text prompt
+ * @param options Generation options (can be NULL for defaults)
+ * @param token_callback Called for each generated token
+ * @param complete_callback Called when generation completes
+ * @param error_callback Called on error
+ * @param user_data User context passed to callbacks
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_process_stream(
+    rac_handle_t handle, const rac_vlm_image_t* image, const char* prompt,
+    const rac_vlm_options_t* options, rac_vlm_component_token_callback_fn token_callback,
+    rac_vlm_component_complete_callback_fn complete_callback,
+    rac_vlm_component_error_callback_fn error_callback, void* user_data);
+
+/**
+ * @brief Get lifecycle state
+ *
+ * @param handle Component handle
+ * @return Current lifecycle state
+ */
+RAC_API rac_lifecycle_state_t rac_vlm_component_get_state(rac_handle_t handle);
+
+/**
+ * @brief Get lifecycle metrics
+ *
+ * @param handle Component handle
+ * @param out_metrics Output: Lifecycle metrics
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_get_metrics(rac_handle_t handle,
+                                                   rac_lifecycle_metrics_t* out_metrics);
+
+/**
+ * @brief Destroy the VLM component
+ *
+ * @param handle Component handle
+ */
+RAC_API void rac_vlm_component_destroy(rac_handle_t handle);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RAC_VLM_COMPONENT_H */

package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_vlm_llamacpp.h

@@ -0,0 +1,216 @@
+/**
+ * @file rac_vlm_llamacpp.h
+ * @brief RunAnywhere Commons - LlamaCPP VLM Backend API
+ *
+ * Public C API for Vision Language Model inference using llama.cpp's
+ * multimodal (mtmd) capabilities. Supports 20+ VLM architectures including
+ * Qwen2-VL, Qwen2.5-VL, SmolVLM, LLaVA, MiniCPM-V, and more.
+ */
+
+#ifndef RAC_VLM_LLAMACPP_H
+#define RAC_VLM_LLAMACPP_H
+
+#include <RACommons/rac_error.h>
+#include <RACommons/rac_types.h>
+#include <RACommons/rac_vlm.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// =============================================================================
+// EXPORT MACRO
+// =============================================================================
+
+#if defined(RAC_LLAMACPP_BUILDING)
+#if defined(_WIN32)
+#define RAC_LLAMACPP_VLM_API __declspec(dllexport)
+#elif defined(__GNUC__) || defined(__clang__)
+#define RAC_LLAMACPP_VLM_API __attribute__((visibility("default")))
+#else
+#define RAC_LLAMACPP_VLM_API
+#endif
+#else
+#define RAC_LLAMACPP_VLM_API
+#endif
+
+// =============================================================================
+// CONFIGURATION
+// =============================================================================
+
+/**
+ * LlamaCPP VLM-specific configuration.
+ */
+typedef struct rac_vlm_llamacpp_config {
+    /** Context size (0 = auto-detect from model) */
+    int32_t context_size;
+
+    /** Number of threads for CPU inference (0 = auto-detect) */
+    int32_t num_threads;
+
+    /** Number of layers to offload to GPU (Metal on iOS/macOS, -1 = all) */
+    int32_t gpu_layers;
+
+    /** Batch size for prompt processing */
+    int32_t batch_size;
+
+    /** Number of threads for vision encoder (0 = same as num_threads) */
+    int32_t vision_threads;
+
+    /** Use GPU for vision encoding */
+    rac_bool_t use_gpu_vision;
+} rac_vlm_llamacpp_config_t;
+
+/**
+ * Default LlamaCPP VLM configuration.
+ */
+static const rac_vlm_llamacpp_config_t RAC_VLM_LLAMACPP_CONFIG_DEFAULT = {
+    .context_size = 0,      // Auto-detect
+    .num_threads = 0,       // Auto-detect
+    .gpu_layers = -1,       // All layers on GPU
+    .batch_size = 512,      //
+    .vision_threads = 0,    // Auto-detect
+    .use_gpu_vision = 1     // Use GPU for vision
+};
+
+// =============================================================================
+// LLAMACPP VLM-SPECIFIC API
+// =============================================================================
+
+/**
+ * Creates a LlamaCPP VLM service.
+ *
+ * @param model_path Path to the GGUF LLM model file
+ * @param mmproj_path Path to the mmproj vision projector GGUF file
+ * @param config LlamaCPP-specific configuration (can be NULL for defaults)
+ * @param out_handle Output: Handle to the created service
+ * @return RAC_SUCCESS or error code
+ */
+RAC_LLAMACPP_VLM_API rac_result_t rac_vlm_llamacpp_create(const char* model_path,
+                                                          const char* mmproj_path,
+                                                          const rac_vlm_llamacpp_config_t* config,
+                                                          rac_handle_t* out_handle);
+
+/**
+ * Loads a VLM model into an existing service.
+ *
+ * @param handle Service handle
+ * @param model_path Path to the GGUF LLM model file
+ * @param mmproj_path Path to the mmproj vision projector GGUF file
+ * @param config LlamaCPP configuration (can be NULL)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_LLAMACPP_VLM_API rac_result_t rac_vlm_llamacpp_load_model(
+    rac_handle_t handle, const char* model_path, const char* mmproj_path,
+    const rac_vlm_llamacpp_config_t* config);
+
+/**
+ * Unloads the current model.
+ *
+ * @param handle Service handle
+ * @return RAC_SUCCESS or error code
+ */
+RAC_LLAMACPP_VLM_API rac_result_t rac_vlm_llamacpp_unload_model(rac_handle_t handle);
+
+/**
+ * Checks if a model is loaded.
+ *
+ * @param handle Service handle
+ * @return RAC_TRUE if model is loaded, RAC_FALSE otherwise
+ */
+RAC_LLAMACPP_VLM_API rac_bool_t rac_vlm_llamacpp_is_model_loaded(rac_handle_t handle);
+
+/**
+ * Processes an image with a text prompt (blocking).
+ *
+ * @param handle Service handle
+ * @param image Image input (file path, RGB pixels, or base64)
+ * @param prompt Text prompt
+ * @param options VLM generation options (can be NULL for defaults)
+ * @param out_result Output: Generation result (caller must free text with rac_free)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_LLAMACPP_VLM_API rac_result_t rac_vlm_llamacpp_process(rac_handle_t handle,
+                                                           const rac_vlm_image_t* image,
+                                                           const char* prompt,
+                                                           const rac_vlm_options_t* options,
+                                                           rac_vlm_result_t* out_result);
+
+/**
+ * Streaming callback for VLM generation.
+ *
+ * @param token Generated token string
+ * @param is_final Whether this is the final token
+ * @param user_data User-provided context
+ * @return RAC_TRUE to continue, RAC_FALSE to stop
+ */
+typedef rac_bool_t (*rac_vlm_llamacpp_stream_callback_fn)(const char* token, rac_bool_t is_final,
+                                                          void* user_data);
+
+/**
+ * Processes an image with streaming callback.
+ *
+ * @param handle Service handle
+ * @param image Image input
+ * @param prompt Text prompt
+ * @param options VLM generation options
+ * @param callback Callback for each token
+ * @param user_data User context passed to callback
+ * @return RAC_SUCCESS or error code
+ */
+RAC_LLAMACPP_VLM_API rac_result_t rac_vlm_llamacpp_process_stream(
+    rac_handle_t handle, const rac_vlm_image_t* image, const char* prompt,
+    const rac_vlm_options_t* options, rac_vlm_llamacpp_stream_callback_fn callback, void* user_data);
+
+/**
+ * Cancels ongoing generation.
+ *
+ * @param handle Service handle
+ */
+RAC_LLAMACPP_VLM_API void rac_vlm_llamacpp_cancel(rac_handle_t handle);
+
+/**
+ * Gets model information as JSON.
+ *
+ * @param handle Service handle
+ * @param out_json Output: JSON string (caller must free with rac_free)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_LLAMACPP_VLM_API rac_result_t rac_vlm_llamacpp_get_model_info(rac_handle_t handle,
+                                                                  char** out_json);
+
+/**
+ * Destroys a LlamaCPP VLM service.
+ *
+ * @param handle Service handle to destroy
+ */
+RAC_LLAMACPP_VLM_API void rac_vlm_llamacpp_destroy(rac_handle_t handle);
+
+// =============================================================================
+// BACKEND REGISTRATION
+// =============================================================================
+
+/**
+ * Registers the LlamaCPP VLM backend with the commons module and service registries.
+ *
+ * Should be called once during SDK initialization.
+ * This registers:
+ * - Module: "llamacpp_vlm" with VISION_LANGUAGE capability
+ * - Service provider: LlamaCPP VLM provider (priority 100)
+ *
+ * @return RAC_SUCCESS or error code
+ */
+RAC_LLAMACPP_VLM_API rac_result_t rac_backend_llamacpp_vlm_register(void);
+
+/**
+ * Unregisters the LlamaCPP VLM backend.
+ *
+ * @return RAC_SUCCESS or error code
+ */
+RAC_LLAMACPP_VLM_API rac_result_t rac_backend_llamacpp_vlm_unregister(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RAC_VLM_LLAMACPP_H */

package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_vlm_service.h

@@ -0,0 +1,206 @@
+/**
+ * @file rac_vlm_service.h
+ * @brief RunAnywhere Commons - VLM Service Interface
+ *
+ * Defines the generic VLM service API and vtable for multi-backend dispatch.
+ * Backends (LlamaCpp VLM, MLX VLM) implement the vtable and register
+ * with the service registry.
+ */
+
+#ifndef RAC_VLM_SERVICE_H
+#define RAC_VLM_SERVICE_H
+
+#include <RACommons/rac_error.h>
+#include <RACommons/rac_vlm_types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// =============================================================================
+// SERVICE VTABLE - Backend implementations provide this
+// =============================================================================
+
+/**
+ * VLM Service operations vtable.
+ * Each backend implements these functions and provides a static vtable.
+ */
+typedef struct rac_vlm_service_ops {
+    /**
+     * Initialize the service with model path(s).
+     * @param impl Backend implementation handle
+     * @param model_path Path to the main model file (LLM weights)
+     * @param mmproj_path Path to vision projector (required for llama.cpp, NULL for MLX)
+     * @return RAC_SUCCESS or error code
+     */
+    rac_result_t (*initialize)(void* impl, const char* model_path, const char* mmproj_path);
+
+    /**
+     * Process an image with a text prompt (blocking).
+     * @param impl Backend implementation handle
+     * @param image Image input
+     * @param prompt Text prompt
+     * @param options Generation options (can be NULL for defaults)
+     * @param out_result Output result (caller must free with rac_vlm_result_free)
+     * @return RAC_SUCCESS or error code
+     */
+    rac_result_t (*process)(void* impl, const rac_vlm_image_t* image, const char* prompt,
+                            const rac_vlm_options_t* options, rac_vlm_result_t* out_result);
+
+    /**
+     * Process an image with streaming callback.
+     * @param impl Backend implementation handle
+     * @param image Image input
+     * @param prompt Text prompt
+     * @param options Generation options (can be NULL for defaults)
+     * @param callback Token callback
+     * @param user_data User context for callback
+     * @return RAC_SUCCESS or error code
+     */
+    rac_result_t (*process_stream)(void* impl, const rac_vlm_image_t* image, const char* prompt,
+                                   const rac_vlm_options_t* options,
+                                   rac_vlm_stream_callback_fn callback, void* user_data);
+
+    /**
+     * Get service information.
+     * @param impl Backend implementation handle
+     * @param out_info Output info structure
+     * @return RAC_SUCCESS or error code
+     */
+    rac_result_t (*get_info)(void* impl, rac_vlm_info_t* out_info);
+
+    /**
+     * Cancel ongoing generation.
+     * @param impl Backend implementation handle
+     * @return RAC_SUCCESS or error code
+     */
+    rac_result_t (*cancel)(void* impl);
+
+    /**
+     * Cleanup/unload model (keeps service alive).
+     * @param impl Backend implementation handle
+     * @return RAC_SUCCESS or error code
+     */
+    rac_result_t (*cleanup)(void* impl);
+
+    /**
+     * Destroy the service.
+     * @param impl Backend implementation handle
+     */
+    void (*destroy)(void* impl);
+} rac_vlm_service_ops_t;
+
+/**
+ * VLM Service instance.
+ * Contains vtable pointer and backend-specific implementation.
+ */
+typedef struct rac_vlm_service {
+    /** Vtable with backend operations */
+    const rac_vlm_service_ops_t* ops;
+
+    /** Backend-specific implementation handle */
+    void* impl;
+
+    /** Model ID for reference */
+    const char* model_id;
+} rac_vlm_service_t;
+
+// =============================================================================
+// PUBLIC API - Generic service functions
+// =============================================================================
+
+/**
+ * @brief Create a VLM service
+ *
+ * Routes through service registry to find appropriate backend.
+ *
+ * @param model_id Model identifier (registry ID or path to model file)
+ * @param out_handle Output: Handle to the created service
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_create(const char* model_id, rac_handle_t* out_handle);
+
+/**
+ * @brief Initialize a VLM service with model paths
+ *
+ * @param handle Service handle
+ * @param model_path Path to the main model file
+ * @param mmproj_path Path to vision projector (can be NULL for some backends)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_initialize(rac_handle_t handle, const char* model_path,
+                                        const char* mmproj_path);
+
+/**
+ * @brief Process an image with a text prompt
+ *
+ * @param handle Service handle
+ * @param image Image input
+ * @param prompt Text prompt describing what to analyze
+ * @param options Generation options (can be NULL for defaults)
+ * @param out_result Output: Generation result (caller must free with rac_vlm_result_free)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_process(rac_handle_t handle, const rac_vlm_image_t* image,
+                                     const char* prompt, const rac_vlm_options_t* options,
+                                     rac_vlm_result_t* out_result);
+
+/**
+ * @brief Process an image with streaming response
+ *
+ * @param handle Service handle
+ * @param image Image input
+ * @param prompt Text prompt
+ * @param options Generation options (can be NULL for defaults)
+ * @param callback Callback for each generated token
+ * @param user_data User context passed to callback
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_process_stream(rac_handle_t handle, const rac_vlm_image_t* image,
+                                            const char* prompt, const rac_vlm_options_t* options,
+                                            rac_vlm_stream_callback_fn callback, void* user_data);
+
+/**
+ * @brief Get service information
+ *
+ * @param handle Service handle
+ * @param out_info Output: Service information
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_get_info(rac_handle_t handle, rac_vlm_info_t* out_info);
+
+/**
+ * @brief Cancel ongoing generation
+ *
+ * @param handle Service handle
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_cancel(rac_handle_t handle);
+
+/**
+ * @brief Cleanup and release model resources
+ *
+ * @param handle Service handle
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_cleanup(rac_handle_t handle);
+
+/**
+ * @brief Destroy a VLM service instance
+ *
+ * @param handle Service handle to destroy
+ */
+RAC_API void rac_vlm_destroy(rac_handle_t handle);
+
+/**
+ * @brief Free a VLM result
+ *
+ * @param result Result to free
+ */
+RAC_API void rac_vlm_result_free(rac_vlm_result_t* result);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RAC_VLM_SERVICE_H */