@runanywhere/core 0.17.7 → 0.18.0

package/android/src/main/include/rac/features/vlm/rac_vlm.h

@@ -0,0 +1,16 @@
+/**
+ * @file rac_vlm.h
+ * @brief RunAnywhere Commons - VLM Public API
+ *
+ * Convenience header that includes all VLM-related headers.
+ * Use this for complete VLM API access.
+ */
+
+#ifndef RAC_VLM_H
+#define RAC_VLM_H
+
+#include "rac/features/vlm/rac_vlm_component.h"
+#include "rac/features/vlm/rac_vlm_service.h"
+#include "rac/features/vlm/rac_vlm_types.h"
+
+#endif /* RAC_VLM_H */

package/android/src/main/include/rac/features/vlm/rac_vlm_component.h

@@ -0,0 +1,168 @@
+/**
+ * @file rac_vlm_component.h
+ * @brief RunAnywhere Commons - VLM Capability Component
+ *
+ * Actor-based VLM capability that owns model lifecycle and generation.
+ * Uses lifecycle manager for unified lifecycle + analytics handling.
+ */
+
+#ifndef RAC_VLM_COMPONENT_H
+#define RAC_VLM_COMPONENT_H
+
+#include "rac/core/capabilities/rac_lifecycle.h"
+#include "rac/core/rac_error.h"
+#include "rac/features/vlm/rac_vlm_types.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// =============================================================================
+// VLM COMPONENT API
+// =============================================================================
+
+/**
+ * @brief Create a VLM capability component
+ *
+ * @param out_handle Output: Handle to the component
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_create(rac_handle_t* out_handle);
+
+/**
+ * @brief Configure the VLM component
+ *
+ * @param handle Component handle
+ * @param config Configuration
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_configure(rac_handle_t handle,
+                                                 const rac_vlm_config_t* config);
+
+/**
+ * @brief Check if model is loaded
+ *
+ * @param handle Component handle
+ * @return RAC_TRUE if loaded, RAC_FALSE otherwise
+ */
+RAC_API rac_bool_t rac_vlm_component_is_loaded(rac_handle_t handle);
+
+/**
+ * @brief Get current model ID
+ *
+ * @param handle Component handle
+ * @return Current model ID (NULL if not loaded)
+ */
+RAC_API const char* rac_vlm_component_get_model_id(rac_handle_t handle);
+
+/**
+ * @brief Load a VLM model
+ *
+ * @param handle Component handle
+ * @param model_path File path to the main model (LLM weights) - REQUIRED
+ * @param mmproj_path File path to the vision projector (required for llama.cpp, NULL for MLX)
+ * @param model_id Model identifier for telemetry (optional: if NULL, defaults to model_path)
+ * @param model_name Human-readable model name (optional: if NULL, defaults to model_id)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_load_model(rac_handle_t handle, const char* model_path,
+                                                  const char* mmproj_path, const char* model_id,
+                                                  const char* model_name);
+
+/**
+ * @brief Unload the current model
+ *
+ * @param handle Component handle
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_unload(rac_handle_t handle);
+
+/**
+ * @brief Cleanup and reset the component
+ *
+ * @param handle Component handle
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_cleanup(rac_handle_t handle);
+
+/**
+ * @brief Cancel ongoing generation
+ *
+ * Best-effort cancellation.
+ *
+ * @param handle Component handle
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_cancel(rac_handle_t handle);
+
+/**
+ * @brief Process an image with text prompt (non-streaming)
+ *
+ * @param handle Component handle
+ * @param image Image input
+ * @param prompt Text prompt
+ * @param options Generation options (can be NULL for defaults)
+ * @param out_result Output: Generation result
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_process(rac_handle_t handle, const rac_vlm_image_t* image,
+                                               const char* prompt, const rac_vlm_options_t* options,
+                                               rac_vlm_result_t* out_result);
+
+/**
+ * @brief Check if streaming is supported
+ *
+ * @param handle Component handle
+ * @return RAC_TRUE if streaming supported, RAC_FALSE otherwise
+ */
+RAC_API rac_bool_t rac_vlm_component_supports_streaming(rac_handle_t handle);
+
+/**
+ * @brief Process an image with streaming
+ *
+ * @param handle Component handle
+ * @param image Image input
+ * @param prompt Text prompt
+ * @param options Generation options (can be NULL for defaults)
+ * @param token_callback Called for each generated token
+ * @param complete_callback Called when generation completes
+ * @param error_callback Called on error
+ * @param user_data User context passed to callbacks
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_process_stream(
+    rac_handle_t handle, const rac_vlm_image_t* image, const char* prompt,
+    const rac_vlm_options_t* options, rac_vlm_component_token_callback_fn token_callback,
+    rac_vlm_component_complete_callback_fn complete_callback,
+    rac_vlm_component_error_callback_fn error_callback, void* user_data);
+
+/**
+ * @brief Get lifecycle state
+ *
+ * @param handle Component handle
+ * @return Current lifecycle state
+ */
+RAC_API rac_lifecycle_state_t rac_vlm_component_get_state(rac_handle_t handle);
+
+/**
+ * @brief Get lifecycle metrics
+ *
+ * @param handle Component handle
+ * @param out_metrics Output: Lifecycle metrics
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_component_get_metrics(rac_handle_t handle,
+                                                   rac_lifecycle_metrics_t* out_metrics);
+
+/**
+ * @brief Destroy the VLM component
+ *
+ * @param handle Component handle
+ */
+RAC_API void rac_vlm_component_destroy(rac_handle_t handle);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RAC_VLM_COMPONENT_H */

package/android/src/main/include/rac/features/vlm/rac_vlm_service.h

@@ -0,0 +1,206 @@
+/**
+ * @file rac_vlm_service.h
+ * @brief RunAnywhere Commons - VLM Service Interface
+ *
+ * Defines the generic VLM service API and vtable for multi-backend dispatch.
+ * Backends (LlamaCpp VLM, MLX VLM) implement the vtable and register
+ * with the service registry.
+ */
+
+#ifndef RAC_VLM_SERVICE_H
+#define RAC_VLM_SERVICE_H
+
+#include "rac/core/rac_error.h"
+#include "rac/features/vlm/rac_vlm_types.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// =============================================================================
+// SERVICE VTABLE - Backend implementations provide this
+// =============================================================================
+
+/**
+ * VLM Service operations vtable.
+ * Each backend implements these functions and provides a static vtable.
+ */
+typedef struct rac_vlm_service_ops {
+    /**
+     * Initialize the service with model path(s).
+     * @param impl Backend implementation handle
+     * @param model_path Path to the main model file (LLM weights)
+     * @param mmproj_path Path to vision projector (required for llama.cpp, NULL for MLX)
+     * @return RAC_SUCCESS or error code
+     */
+    rac_result_t (*initialize)(void* impl, const char* model_path, const char* mmproj_path);
+
+    /**
+     * Process an image with a text prompt (blocking).
+     * @param impl Backend implementation handle
+     * @param image Image input
+     * @param prompt Text prompt
+     * @param options Generation options (can be NULL for defaults)
+     * @param out_result Output result (caller must free with rac_vlm_result_free)
+     * @return RAC_SUCCESS or error code
+     */
+    rac_result_t (*process)(void* impl, const rac_vlm_image_t* image, const char* prompt,
+                            const rac_vlm_options_t* options, rac_vlm_result_t* out_result);
+
+    /**
+     * Process an image with streaming callback.
+     * @param impl Backend implementation handle
+     * @param image Image input
+     * @param prompt Text prompt
+     * @param options Generation options (can be NULL for defaults)
+     * @param callback Token callback
+     * @param user_data User context for callback
+     * @return RAC_SUCCESS or error code
+     */
+    rac_result_t (*process_stream)(void* impl, const rac_vlm_image_t* image, const char* prompt,
+                                   const rac_vlm_options_t* options,
+                                   rac_vlm_stream_callback_fn callback, void* user_data);
+
+    /**
+     * Get service information.
+     * @param impl Backend implementation handle
+     * @param out_info Output info structure
+     * @return RAC_SUCCESS or error code
+     */
+    rac_result_t (*get_info)(void* impl, rac_vlm_info_t* out_info);
+
+    /**
+     * Cancel ongoing generation.
+     * @param impl Backend implementation handle
+     * @return RAC_SUCCESS or error code
+     */
+    rac_result_t (*cancel)(void* impl);
+
+    /**
+     * Cleanup/unload model (keeps service alive).
+     * @param impl Backend implementation handle
+     * @return RAC_SUCCESS or error code
+     */
+    rac_result_t (*cleanup)(void* impl);
+
+    /**
+     * Destroy the service.
+     * @param impl Backend implementation handle
+     */
+    void (*destroy)(void* impl);
+} rac_vlm_service_ops_t;
+
+/**
+ * VLM Service instance.
+ * Contains vtable pointer and backend-specific implementation.
+ */
+typedef struct rac_vlm_service {
+    /** Vtable with backend operations */
+    const rac_vlm_service_ops_t* ops;
+
+    /** Backend-specific implementation handle */
+    void* impl;
+
+    /** Model ID for reference */
+    const char* model_id;
+} rac_vlm_service_t;
+
+// =============================================================================
+// PUBLIC API - Generic service functions
+// =============================================================================
+
+/**
+ * @brief Create a VLM service
+ *
+ * Routes through service registry to find appropriate backend.
+ *
+ * @param model_id Model identifier (registry ID or path to model file)
+ * @param out_handle Output: Handle to the created service
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_create(const char* model_id, rac_handle_t* out_handle);
+
+/**
+ * @brief Initialize a VLM service with model paths
+ *
+ * @param handle Service handle
+ * @param model_path Path to the main model file
+ * @param mmproj_path Path to vision projector (can be NULL for some backends)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_initialize(rac_handle_t handle, const char* model_path,
+                                        const char* mmproj_path);
+
+/**
+ * @brief Process an image with a text prompt
+ *
+ * @param handle Service handle
+ * @param image Image input
+ * @param prompt Text prompt describing what to analyze
+ * @param options Generation options (can be NULL for defaults)
+ * @param out_result Output: Generation result (caller must free with rac_vlm_result_free)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_process(rac_handle_t handle, const rac_vlm_image_t* image,
+                                     const char* prompt, const rac_vlm_options_t* options,
+                                     rac_vlm_result_t* out_result);
+
+/**
+ * @brief Process an image with streaming response
+ *
+ * @param handle Service handle
+ * @param image Image input
+ * @param prompt Text prompt
+ * @param options Generation options (can be NULL for defaults)
+ * @param callback Callback for each generated token
+ * @param user_data User context passed to callback
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_process_stream(rac_handle_t handle, const rac_vlm_image_t* image,
+                                            const char* prompt, const rac_vlm_options_t* options,
+                                            rac_vlm_stream_callback_fn callback, void* user_data);
+
+/**
+ * @brief Get service information
+ *
+ * @param handle Service handle
+ * @param out_info Output: Service information
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_get_info(rac_handle_t handle, rac_vlm_info_t* out_info);
+
+/**
+ * @brief Cancel ongoing generation
+ *
+ * @param handle Service handle
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_cancel(rac_handle_t handle);
+
+/**
+ * @brief Cleanup and release model resources
+ *
+ * @param handle Service handle
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_vlm_cleanup(rac_handle_t handle);
+
+/**
+ * @brief Destroy a VLM service instance
+ *
+ * @param handle Service handle to destroy
+ */
+RAC_API void rac_vlm_destroy(rac_handle_t handle);
+
+/**
+ * @brief Free a VLM result
+ *
+ * @param result Result to free
+ */
+RAC_API void rac_vlm_result_free(rac_vlm_result_t* result);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RAC_VLM_SERVICE_H */