@runanywhere/core 0.17.7 → 0.18.0

package/android/src/main/include/rac/features/vlm/rac_vlm_types.h

@@ -0,0 +1,417 @@
+/**
+ * @file rac_vlm_types.h
+ * @brief RunAnywhere Commons - VLM Types and Data Structures
+ *
+ * Defines data structures for Vision Language Model (VLM) operations.
+ * Supports image input (file path, RGB pixels, base64), generation options,
+ * results, and streaming callbacks.
+ *
+ * For the service interface, see rac_vlm_service.h.
+ */
+
+#ifndef RAC_VLM_TYPES_H
+#define RAC_VLM_TYPES_H
+
+#include "rac/core/rac_types.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// =============================================================================
+// CHAT TEMPLATE - Abstraction for VLM prompt formatting
+// =============================================================================
+
+/**
+ * @brief Known VLM model families for chat template selection
+ *
+ * Use RAC_VLM_MODEL_FAMILY_AUTO (default) to auto-detect from model metadata.
+ * Use RAC_VLM_MODEL_FAMILY_CUSTOM with a custom template string for new models.
+ *
+ * Verified templates (from official HuggingFace repos):
+ * - QWEN2_VL: <|im_start|>system\nYou are a helpful assistant.<|im_end|>\n
+ *             <|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{prompt}<|im_end|>\n
+ *             <|im_start|>assistant\n
+ * - SMOLVLM:  <|im_start|>User: {image}{prompt} \nAssistant:
+ * - LLAVA:    USER: <image>\n{prompt}\nASSISTANT:
+ */
+typedef enum rac_vlm_model_family {
+    RAC_VLM_MODEL_FAMILY_AUTO = 0,     /**< Auto-detect from model metadata (default) */
+    RAC_VLM_MODEL_FAMILY_QWEN2_VL = 1, /**< Qwen2-VL: chatml with <|vision_start|> markers */
+    RAC_VLM_MODEL_FAMILY_SMOLVLM = 2,  /**< SmolVLM: <|im_start|>User: format */
+    RAC_VLM_MODEL_FAMILY_LLAVA = 3,    /**< LLaVA/Vicuna: USER:/ASSISTANT: format */
+    RAC_VLM_MODEL_FAMILY_CUSTOM = 99,  /**< Use custom_chat_template string */
+} rac_vlm_model_family_t;
+
+/**
+ * @brief Custom chat template for VLM prompt formatting
+ *
+ * A simple template string with placeholders:
+ *   {system}  - System prompt (optional, can be empty)
+ *   {image}   - Image marker/placeholder
+ *   {prompt}  - User's text prompt
+ *
+ * Example template string:
+ *   "<|im_start|>user\n{image}{prompt}<|im_end|>\n<|im_start|>assistant\n"
+ *
+ * The SDK will replace placeholders at runtime. If {system} is in the template
+ * but no system prompt is provided, it uses a default or leaves empty.
+ */
+typedef struct rac_vlm_chat_template {
+    /**
+     * Full template string with {system}, {image}, {prompt} placeholders.
+     * Example: "<|im_start|>user\n{image}{prompt}<|im_end|>\n<|im_start|>assistant\n"
+     */
+    const char* template_str;
+
+    /**
+     * Image marker to insert at {image} placeholder.
+     * Examples: "<image>", "<|vision_start|><|image_pad|><|vision_end|>"
+     * If NULL, uses the backend's default marker.
+     */
+    const char* image_marker;
+
+    /**
+     * Default system prompt if {system} is in template but none provided.
+     * Can be NULL for no default.
+     */
+    const char* default_system_prompt;
+} rac_vlm_chat_template_t;
+
+/**
+ * @brief Get built-in chat template for a model family
+ *
+ * @param family Model family enum value
+ * @return Pointer to static template, or NULL if family not supported
+ */
+RAC_API const rac_vlm_chat_template_t* rac_vlm_get_builtin_template(rac_vlm_model_family_t family);
+
+// =============================================================================
+// IMAGE INPUT - Supports multiple input formats
+// =============================================================================
+
+/**
+ * @brief VLM image input format enumeration
+ */
+typedef enum rac_vlm_image_format {
+    RAC_VLM_IMAGE_FORMAT_FILE_PATH = 0,   /**< Path to image file (JPEG, PNG, etc.) */
+    RAC_VLM_IMAGE_FORMAT_RGB_PIXELS = 1,  /**< Raw RGB pixel buffer (RGBRGBRGB...) */
+    RAC_VLM_IMAGE_FORMAT_BASE64 = 2,      /**< Base64-encoded image data */
+} rac_vlm_image_format_t;
+
+/**
+ * @brief VLM image input structure
+ *
+ * Represents an image input for VLM processing. Supports three formats:
+ * - FILE_PATH: Path to an image file on disk
+ * - RGB_PIXELS: Raw RGB pixel data with width/height
+ * - BASE64: Base64-encoded image data
+ */
+typedef struct rac_vlm_image {
+    /** Image format type */
+    rac_vlm_image_format_t format;
+
+    /** Path to image file (for FILE_PATH format) */
+    const char* file_path;
+
+    /** Raw RGB pixel data (for RGB_PIXELS format, layout: RGBRGBRGB...) */
+    const uint8_t* pixel_data;
+
+    /** Base64-encoded image data (for BASE64 format) */
+    const char* base64_data;
+
+    /** Image width in pixels (required for RGB_PIXELS, 0 otherwise) */
+    uint32_t width;
+
+    /** Image height in pixels (required for RGB_PIXELS, 0 otherwise) */
+    uint32_t height;
+
+    /** Size of pixel_data or base64_data in bytes */
+    size_t data_size;
+} rac_vlm_image_t;
+
+// =============================================================================
+// OPTIONS - VLM Generation Options
+// =============================================================================
+
+/**
+ * @brief VLM generation options
+ *
+ * Controls text generation behavior for VLM inference.
+ * Combines standard LLM options with VLM-specific parameters.
+ */
+typedef struct rac_vlm_options {
+    // ── Standard Generation Parameters ──
+    /** Maximum number of tokens to generate (default: 2048) */
+    int32_t max_tokens;
+
+    /** Temperature for sampling (0.0 - 2.0, default: 0.7) */
+    float temperature;
+
+    /** Top-p sampling parameter (default: 0.9) */
+    float top_p;
+
+    /** Stop sequences (null-terminated array, can be NULL) */
+    const char* const* stop_sequences;
+
+    /** Number of stop sequences */
+    size_t num_stop_sequences;
+
+    /** Enable streaming mode (default: true) */
+    rac_bool_t streaming_enabled;
+
+    /** System prompt (can be NULL, uses template default if available) */
+    const char* system_prompt;
+
+    // ── VLM-Specific Parameters ──
+    /** Max image dimension for resize (0 = model default) */
+    int32_t max_image_size;
+
+    /** Number of CPU threads for vision encoder (0 = auto) */
+    int32_t n_threads;
+
+    /** Use GPU for vision encoding */
+    rac_bool_t use_gpu;
+
+    // ── Chat Template Configuration ──
+    /**
+     * Model family for automatic chat template selection.
+     * Set to RAC_VLM_MODEL_FAMILY_AUTO (default) to auto-detect from model metadata.
+     * Set to RAC_VLM_MODEL_FAMILY_CUSTOM and provide custom_chat_template for custom templates.
+     */
+    rac_vlm_model_family_t model_family;
+
+    /**
+     * Custom chat template (only used when model_family == RAC_VLM_MODEL_FAMILY_CUSTOM).
+     * If NULL and model_family is CUSTOM, falls back to GENERIC template.
+     */
+    const rac_vlm_chat_template_t* custom_chat_template;
+
+    /**
+     * Override image marker (can be NULL to use template default).
+     * Useful when the default marker doesn't match your model's expectations.
+     */
+    const char* image_marker_override;
+} rac_vlm_options_t;
+
+/**
+ * @brief Default VLM generation options
+ */
+#define RAC_VLM_OPTIONS_DEFAULT                                                                    \
+    {                                                                                              \
+        .max_tokens = 2048, .temperature = 0.7f, .top_p = 0.9f, .stop_sequences = RAC_NULL,        \
+        .num_stop_sequences = 0, .streaming_enabled = RAC_TRUE, .system_prompt = RAC_NULL,         \
+        .max_image_size = 0, .n_threads = 0, .use_gpu = RAC_TRUE,                                  \
+        .model_family = RAC_VLM_MODEL_FAMILY_AUTO, .custom_chat_template = RAC_NULL,               \
+        .image_marker_override = RAC_NULL                                                          \
+    }
+
+// =============================================================================
+// CONFIGURATION - VLM Component Configuration
+// =============================================================================
+
+/**
+ * @brief VLM component configuration
+ *
+ * Configuration for initializing a VLM component.
+ */
+typedef struct rac_vlm_config {
+    /** Model ID (optional - uses default if NULL) */
+    const char* model_id;
+
+    /** Preferred framework for generation (use RAC_FRAMEWORK_UNKNOWN for auto) */
+    int32_t preferred_framework;
+
+    /** Context length - max tokens the model can handle (default: 4096) */
+    int32_t context_length;
+
+    /** Temperature for sampling (0.0 - 2.0, default: 0.7) */
+    float temperature;
+
+    /** Maximum tokens to generate (default: 2048) */
+    int32_t max_tokens;
+
+    /** System prompt for generation (can be NULL) */
+    const char* system_prompt;
+
+    /** Enable streaming mode (default: true) */
+    rac_bool_t streaming_enabled;
+} rac_vlm_config_t;
+
+/**
+ * @brief Default VLM configuration
+ */
+static const rac_vlm_config_t RAC_VLM_CONFIG_DEFAULT = {.model_id = RAC_NULL,
+                                                        .preferred_framework =
+                                                            99,  // RAC_FRAMEWORK_UNKNOWN
+                                                        .context_length = 4096,
+                                                        .temperature = 0.7f,
+                                                        .max_tokens = 2048,
+                                                        .system_prompt = RAC_NULL,
+                                                        .streaming_enabled = RAC_TRUE};
+
+// =============================================================================
+// RESULTS - VLM Generation Results
+// =============================================================================
+
+/**
+ * @brief VLM generation result
+ *
+ * Contains the generated text and detailed metrics for VLM inference.
+ */
+typedef struct rac_vlm_result {
+    /** Generated text (owned, must be freed with rac_vlm_result_free) */
+    char* text;
+
+    /** Number of tokens in prompt (including text tokens) */
+    int32_t prompt_tokens;
+
+    /** Number of vision/image tokens specifically */
+    int32_t image_tokens;
+
+    /** Number of tokens generated */
+    int32_t completion_tokens;
+
+    /** Total tokens (prompt + completion) */
+    int32_t total_tokens;
+
+    /** Time to first token in milliseconds */
+    int64_t time_to_first_token_ms;
+
+    /** Time spent encoding the image in milliseconds */
+    int64_t image_encode_time_ms;
+
+    /** Total generation time in milliseconds */
+    int64_t total_time_ms;
+
+    /** Tokens generated per second */
+    float tokens_per_second;
+} rac_vlm_result_t;
+
+// =============================================================================
+// SERVICE INFO - VLM Service Information
+// =============================================================================
+
+/**
+ * @brief VLM service handle info
+ *
+ * Provides information about a VLM service instance.
+ */
+typedef struct rac_vlm_info {
+    /** Whether the service is ready for generation */
+    rac_bool_t is_ready;
+
+    /** Current model identifier (can be NULL if not loaded) */
+    const char* current_model;
+
+    /** Context length (0 if unknown) */
+    int32_t context_length;
+
+    /** Whether streaming is supported */
+    rac_bool_t supports_streaming;
+
+    /** Whether multiple images per request are supported */
+    rac_bool_t supports_multiple_images;
+
+    /** Vision encoder type ("clip", "siglip", "fastvithd", etc.) */
+    const char* vision_encoder_type;
+} rac_vlm_info_t;
+
+// =============================================================================
+// CALLBACKS - Streaming Callbacks
+// =============================================================================
+
+/**
+ * @brief Simple VLM streaming callback
+ *
+ * Called for each generated token during streaming.
+ *
+ * @param token The generated token string
+ * @param user_data User-provided context
+ * @return RAC_TRUE to continue, RAC_FALSE to stop generation
+ */
+typedef rac_bool_t (*rac_vlm_stream_callback_fn)(const char* token, void* user_data);
+
+/**
+ * @brief Extended token event structure
+ *
+ * Provides detailed information about each token during streaming.
+ */
+typedef struct rac_vlm_token_event {
+    /** The generated token text */
+    const char* token;
+
+    /** Token index in the sequence */
+    int32_t token_index;
+
+    /** Is this the final token? */
+    rac_bool_t is_final;
+
+    /** Tokens generated per second so far */
+    float tokens_per_second;
+} rac_vlm_token_event_t;
+
+/**
+ * @brief Extended streaming callback with token event details
+ *
+ * @param event Token event details
+ * @param user_data User-provided context
+ * @return RAC_TRUE to continue, RAC_FALSE to stop generation
+ */
+typedef rac_bool_t (*rac_vlm_token_event_callback_fn)(const rac_vlm_token_event_t* event,
+                                                      void* user_data);
+
+// =============================================================================
+// COMPONENT CALLBACKS - For component-level streaming
+// =============================================================================
+
+/**
+ * @brief VLM component token callback
+ *
+ * @param token The generated token
+ * @param user_data User-provided context
+ * @return RAC_TRUE to continue, RAC_FALSE to stop
+ */
+typedef rac_bool_t (*rac_vlm_component_token_callback_fn)(const char* token, void* user_data);
+
+/**
+ * @brief VLM component completion callback
+ *
+ * Called when streaming is complete with final result.
+ *
+ * @param result Final generation result with metrics
+ * @param user_data User-provided context
+ */
+typedef void (*rac_vlm_component_complete_callback_fn)(const rac_vlm_result_t* result,
+                                                       void* user_data);
+
+/**
+ * @brief VLM component error callback
+ *
+ * Called if streaming fails.
+ *
+ * @param error_code Error code
+ * @param error_message Error message
+ * @param user_data User-provided context
+ */
+typedef void (*rac_vlm_component_error_callback_fn)(rac_result_t error_code,
+                                                    const char* error_message, void* user_data);
+
+// =============================================================================
+// MEMORY MANAGEMENT
+// =============================================================================
+
+/**
+ * @brief Free VLM result resources
+ *
+ * Frees the text and any other owned resources in the result.
+ *
+ * @param result Result to free (can be NULL)
+ */
+RAC_API void rac_vlm_result_free(rac_vlm_result_t* result);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RAC_VLM_TYPES_H */

package/android/src/main/include/rac/infrastructure/model_management/rac_model_registry.h

@@ -86,6 +86,21 @@ RAC_API rac_result_t rac_model_registry_save(rac_model_registry_handle_t handle,
 RAC_API rac_result_t rac_model_registry_get(rac_model_registry_handle_t handle,
                                             const char* model_id, rac_model_info_t** out_model);
 
+/**
+ * @brief Get model metadata by local path.
+ *
+ * Searches through all registered models and returns the one with matching local_path.
+ * This is useful when loading models by path instead of model_id.
+ *
+ * @param handle Registry handle
+ * @param local_path Local path to search for
+ * @param out_model Output: Model info (owned, must be freed with rac_model_info_free)
+ * @return RAC_SUCCESS, RAC_ERROR_NOT_FOUND, or other error code
+ */
+RAC_API rac_result_t rac_model_registry_get_by_path(rac_model_registry_handle_t handle,
+                                                    const char* local_path,
+                                                    rac_model_info_t** out_model);
+
 /**
  * @brief Load all stored models.
  *

package/android/src/main/include/rac/infrastructure/model_management/rac_model_types.h

@@ -163,6 +163,7 @@ typedef enum rac_model_format {
     RAC_MODEL_FORMAT_ORT = 1,     /**< ONNX Runtime format */
     RAC_MODEL_FORMAT_GGUF = 2,    /**< GGUF format (llama.cpp) */
     RAC_MODEL_FORMAT_BIN = 3,     /**< Binary format */
+    RAC_MODEL_FORMAT_COREML = 4,  /**< Core ML format (.mlmodelc, .mlpackage) */
     RAC_MODEL_FORMAT_UNKNOWN = 99 /**< Unknown format */
 } rac_model_format_t;
 
@@ -182,6 +183,8 @@ typedef enum rac_inference_framework {
     RAC_FRAMEWORK_FLUID_AUDIO = 4,       /**< FluidAudio */
     RAC_FRAMEWORK_BUILTIN = 5,           /**< Built-in (e.g., energy VAD) */
     RAC_FRAMEWORK_NONE = 6,              /**< No framework needed */
+    RAC_FRAMEWORK_MLX = 7,               /**< MLX C++ (Apple Silicon VLM) */
+    RAC_FRAMEWORK_COREML = 8,            /**< Core ML (Apple Neural Engine) */
     RAC_FRAMEWORK_UNKNOWN = 99           /**< Unknown framework */
 } rac_inference_framework_t;
 

package/android/src/main/include/rac/utils/rac_image_utils.h

@@ -0,0 +1,215 @@
+/**
+ * @file rac_image_utils.h
+ * @brief RunAnywhere Commons - Image Utilities
+ *
+ * Image loading and processing utilities for VLM backends.
+ * Supports loading from file paths, decoding base64, and resizing.
+ */
+
+#ifndef RAC_IMAGE_UTILS_H
+#define RAC_IMAGE_UTILS_H
+
+#include "rac/core/rac_error.h"
+#include "rac/core/rac_types.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// =============================================================================
+// IMAGE DATA STRUCTURES
+// =============================================================================
+
+/**
+ * @brief Loaded image data
+ *
+ * Contains RGB pixel data after loading an image.
+ * Must be freed with rac_image_free().
+ */
+typedef struct rac_image_data {
+    /** Raw RGB pixel data (RGBRGBRGB...) */
+    uint8_t* pixels;
+
+    /** Image width in pixels */
+    int32_t width;
+
+    /** Image height in pixels */
+    int32_t height;
+
+    /** Number of channels (3 for RGB) */
+    int32_t channels;
+
+    /** Total size in bytes (width * height * channels) */
+    size_t size;
+} rac_image_data_t;
+
+/**
+ * @brief Normalized float image data
+ *
+ * Contains normalized float32 pixel data (values in [-1, 1] or [0, 1]).
+ * Used by vision encoders.
+ */
+typedef struct rac_image_float {
+    /** Normalized float pixel data */
+    float* pixels;
+
+    /** Image width in pixels */
+    int32_t width;
+
+    /** Image height in pixels */
+    int32_t height;
+
+    /** Number of channels (3 for RGB) */
+    int32_t channels;
+
+    /** Total number of floats (width * height * channels) */
+    size_t count;
+} rac_image_float_t;
+
+// =============================================================================
+// IMAGE LOADING
+// =============================================================================
+
+/**
+ * @brief Load an image from a file path
+ *
+ * Supports JPEG, PNG, BMP, GIF, and other common formats via stb_image.
+ * Output is always RGB (3 channels).
+ *
+ * @param file_path Path to the image file
+ * @param out_image Output: Loaded image data (must be freed with rac_image_free)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_image_load_file(const char* file_path, rac_image_data_t* out_image);
+
+/**
+ * @brief Decode a base64-encoded image
+ *
+ * Decodes base64 data and loads the image.
+ * Supports the same formats as rac_image_load_file.
+ *
+ * @param base64_data Base64-encoded image data
+ * @param data_size Length of the base64 string
+ * @param out_image Output: Loaded image data (must be freed with rac_image_free)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_image_decode_base64(const char* base64_data, size_t data_size,
+                                             rac_image_data_t* out_image);
+
+/**
+ * @brief Decode image from raw bytes
+ *
+ * Decodes an image from raw bytes (e.g., from network response).
+ *
+ * @param data Raw image data (JPEG, PNG, etc.)
+ * @param data_size Size of the data in bytes
+ * @param out_image Output: Loaded image data (must be freed with rac_image_free)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_image_decode_bytes(const uint8_t* data, size_t data_size,
+                                            rac_image_data_t* out_image);
+
+// =============================================================================
+// IMAGE PROCESSING
+// =============================================================================
+
+/**
+ * @brief Resize an image
+ *
+ * Resizes the image to the specified dimensions using bilinear interpolation.
+ *
+ * @param image Input image
+ * @param new_width Target width
+ * @param new_height Target height
+ * @param out_image Output: Resized image (must be freed with rac_image_free)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_image_resize(const rac_image_data_t* image, int32_t new_width,
+                                      int32_t new_height, rac_image_data_t* out_image);
+
+/**
+ * @brief Resize an image maintaining aspect ratio
+ *
+ * Resizes the image so that the longest dimension equals max_size.
+ * Aspect ratio is preserved.
+ *
+ * @param image Input image
+ * @param max_size Maximum dimension (width or height)
+ * @param out_image Output: Resized image (must be freed with rac_image_free)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_image_resize_max(const rac_image_data_t* image, int32_t max_size,
+                                          rac_image_data_t* out_image);
+
+/**
+ * @brief Normalize image to float values
+ *
+ * Converts uint8 pixels to float32 with optional mean/std normalization.
+ * Commonly used for vision encoders (CLIP, SigLIP, etc.).
+ *
+ * Formula: pixel_normalized = (pixel / 255.0 - mean) / std
+ *
+ * @param image Input image
+ * @param mean Per-channel mean values (array of 3 floats, or NULL for [0,0,0])
+ * @param std Per-channel std values (array of 3 floats, or NULL for [1,1,1])
+ * @param out_float Output: Normalized float image (must be freed with rac_image_float_free)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_image_normalize(const rac_image_data_t* image, const float* mean,
+                                         const float* std, rac_image_float_t* out_float);
+
+/**
+ * @brief Convert RGB to CHW format
+ *
+ * Converts from HWC (Height, Width, Channels) to CHW format.
+ * Many neural networks expect CHW format.
+ *
+ * @param image Input float image in HWC format
+ * @param out_chw Output: Float image in CHW format (must be freed with rac_image_float_free)
+ * @return RAC_SUCCESS or error code
+ */
+RAC_API rac_result_t rac_image_to_chw(const rac_image_float_t* image, rac_image_float_t* out_chw);
+
+// =============================================================================
+// MEMORY MANAGEMENT
+// =============================================================================
+
+/**
+ * @brief Free image data
+ *
+ * Frees the pixel data allocated by image loading functions.
+ *
+ * @param image Image to free (can be NULL)
+ */
+RAC_API void rac_image_free(rac_image_data_t* image);
+
+/**
+ * @brief Free float image data
+ *
+ * Frees the pixel data allocated by normalization functions.
+ *
+ * @param image Float image to free (can be NULL)
+ */
+RAC_API void rac_image_float_free(rac_image_float_t* image);
+
+// =============================================================================
+// UTILITY FUNCTIONS
+// =============================================================================
+
+/**
+ * @brief Calculate resized dimensions maintaining aspect ratio
+ *
+ * @param width Original width
+ * @param height Original height
+ * @param max_size Maximum dimension
+ * @param out_width Output: New width
+ * @param out_height Output: New height
+ */
+RAC_API void rac_image_calc_resize(int32_t width, int32_t height, int32_t max_size,
+                                   int32_t* out_width, int32_t* out_height);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RAC_IMAGE_UTILS_H */