@runanywhere/core 0.17.7 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/README.md +218 -2
  2. package/RunAnywhereCore.podspec +1 -0
  3. package/android/CMakeLists.txt +24 -2
  4. package/android/build.gradle +61 -9
  5. package/android/src/main/cpp/cpp-adapter.cpp +51 -3
  6. package/android/src/main/include/rac/backends/rac_vlm_llamacpp.h +216 -0
  7. package/android/src/main/include/rac/core/capabilities/rac_lifecycle.h +3 -1
  8. package/android/src/main/include/rac/core/rac_core.h +11 -0
  9. package/android/src/main/include/rac/core/rac_types.h +8 -6
  10. package/android/src/main/include/rac/features/diffusion/rac_diffusion.h +22 -0
  11. package/android/src/main/include/rac/features/diffusion/rac_diffusion_component.h +263 -0
  12. package/android/src/main/include/rac/features/diffusion/rac_diffusion_model_registry.h +358 -0
  13. package/android/src/main/include/rac/features/diffusion/rac_diffusion_service.h +187 -0
  14. package/android/src/main/include/rac/features/diffusion/rac_diffusion_tokenizer.h +167 -0
  15. package/android/src/main/include/rac/features/diffusion/rac_diffusion_types.h +454 -0
  16. package/android/src/main/include/rac/features/llm/rac_tool_calling.h +373 -0
  17. package/android/src/main/include/rac/features/platform/rac_diffusion_platform.h +305 -0
  18. package/android/src/main/include/rac/features/vad/rac_vad_energy.h +1 -1
  19. package/android/src/main/include/rac/features/vlm/rac_vlm.h +16 -0
  20. package/android/src/main/include/rac/features/vlm/rac_vlm_component.h +168 -0
  21. package/android/src/main/include/rac/features/vlm/rac_vlm_service.h +206 -0
  22. package/android/src/main/include/rac/features/vlm/rac_vlm_types.h +417 -0
  23. package/android/src/main/include/rac/infrastructure/model_management/rac_model_registry.h +15 -0
  24. package/android/src/main/include/rac/infrastructure/model_management/rac_model_types.h +3 -0
  25. package/android/src/main/include/rac/utils/rac_image_utils.h +215 -0
  26. package/android/src/main/java/com/margelo/nitro/runanywhere/PlatformAdapterBridge.kt +201 -1
  27. package/android/src/main/jniLibs/arm64-v8a/libc++_shared.so +0 -0
  28. package/android/src/main/jniLibs/arm64-v8a/libomp.so +0 -0
  29. package/android/src/main/jniLibs/arm64-v8a/librac_commons.so +0 -0
  30. package/android/src/main/jniLibs/arm64-v8a/librunanywhere_jni.so +0 -0
  31. package/android/src/main/jniLibs/x86_64/libc++_shared.so +0 -0
  32. package/android/src/main/jniLibs/x86_64/libomp.so +0 -0
  33. package/android/src/main/jniLibs/x86_64/librac_commons.so +0 -0
  34. package/android/src/main/jniLibs/x86_64/librunanywhere_jni.so +0 -0
  35. package/cpp/HybridRunAnywhereCore.cpp +259 -160
  36. package/cpp/HybridRunAnywhereCore.hpp +11 -0
  37. package/cpp/bridges/InitBridge.cpp +234 -3
  38. package/cpp/bridges/PlatformDownloadBridge.h +44 -0
  39. package/cpp/bridges/ToolCallingBridge.cpp +188 -0
  40. package/cpp/bridges/ToolCallingBridge.hpp +98 -0
  41. package/cpp/third_party/nlohmann/json.hpp +24765 -0
  42. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/RACommons.h +18 -4
  43. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_core.h +11 -0
  44. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_diffusion.h +22 -0
  45. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_diffusion_component.h +263 -0
  46. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_diffusion_model_registry.h +358 -0
  47. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_diffusion_platform.h +305 -0
  48. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_diffusion_service.h +187 -0
  49. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_diffusion_tokenizer.h +167 -0
  50. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_diffusion_types.h +454 -0
  51. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_endpoints.h +3 -17
  52. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_image_utils.h +215 -0
  53. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_lifecycle.h +3 -1
  54. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_model_assignment.h +4 -20
  55. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_model_registry.h +15 -0
  56. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_model_types.h +3 -0
  57. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_tool_calling.h +373 -0
  58. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_types.h +8 -6
  59. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_vad_energy.h +1 -1
  60. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_vlm.h +16 -0
  61. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_vlm_component.h +168 -0
  62. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_vlm_llamacpp.h +216 -0
  63. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_vlm_service.h +206 -0
  64. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_vlm_types.h +417 -0
  65. package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/RACommons +0 -0
  66. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/RACommons.h +18 -4
  67. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_core.h +11 -0
  68. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_diffusion.h +22 -0
  69. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_diffusion_component.h +263 -0
  70. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_diffusion_model_registry.h +358 -0
  71. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_diffusion_platform.h +305 -0
  72. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_diffusion_service.h +187 -0
  73. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_diffusion_tokenizer.h +167 -0
  74. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_diffusion_types.h +454 -0
  75. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_endpoints.h +3 -17
  76. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_image_utils.h +215 -0
  77. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_lifecycle.h +3 -1
  78. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_model_assignment.h +4 -20
  79. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_model_registry.h +15 -0
  80. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_model_types.h +3 -0
  81. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_tool_calling.h +373 -0
  82. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_types.h +8 -6
  83. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_vad_energy.h +1 -1
  84. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_vlm.h +16 -0
  85. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_vlm_component.h +168 -0
  86. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_vlm_llamacpp.h +216 -0
  87. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_vlm_service.h +206 -0
  88. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_vlm_types.h +417 -0
  89. package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/RACommons +0 -0
  90. package/ios/PlatformAdapterBridge.h +24 -1
  91. package/ios/PlatformAdapterBridge.m +243 -0
  92. package/nitrogen/generated/shared/c++/HybridRunAnywhereCoreSpec.cpp +4 -0
  93. package/nitrogen/generated/shared/c++/HybridRunAnywhereCoreSpec.hpp +4 -0
  94. package/package.json +8 -4
  95. package/src/Foundation/Security/SecureStorageService.ts +12 -6
  96. package/src/Public/Extensions/RunAnywhere+Models.ts +5 -3
  97. package/src/Public/Extensions/RunAnywhere+STT.ts +7 -2
  98. package/src/Public/Extensions/RunAnywhere+ToolCalling.ts +472 -0
  99. package/src/Public/Extensions/index.ts +16 -0
  100. package/src/Public/RunAnywhere.ts +18 -0
  101. package/src/index.ts +0 -1
  102. package/src/services/Network/index.ts +0 -1
  103. package/src/services/index.ts +0 -1
  104. package/src/specs/RunAnywhereCore.nitro.ts +72 -0
  105. package/src/types/ToolCallingTypes.ts +198 -0
  106. package/src/types/index.ts +13 -0
@@ -0,0 +1,454 @@
1
+ /**
2
+ * @file rac_diffusion_types.h
3
+ * @brief RunAnywhere Commons - Diffusion Types and Data Structures
4
+ *
5
+ * This header defines data structures for image generation using diffusion models
6
+ * (Stable Diffusion). Supports text-to-image, image-to-image, and inpainting.
7
+ *
8
+ * This header defines data structures only. For the service interface,
9
+ * see rac_diffusion_service.h.
10
+ */
11
+
12
+ #ifndef RAC_DIFFUSION_TYPES_H
13
+ #define RAC_DIFFUSION_TYPES_H
14
+
15
+ #include <RACommons/rac_types.h>
16
+
17
+ #ifdef __cplusplus
18
+ extern "C" {
19
+ #endif
20
+
21
+ // =============================================================================
22
+ // SCHEDULER TYPES
23
+ // =============================================================================
24
+
25
+ /**
26
+ * @brief Diffusion scheduler/sampler types
27
+ *
28
+ * Different scheduling algorithms for the denoising process.
29
+ * DPM++ 2M Karras is recommended for best quality/speed tradeoff.
30
+ */
31
+ typedef enum rac_diffusion_scheduler {
32
+ RAC_DIFFUSION_SCHEDULER_DPM_PP_2M_KARRAS = 0, /**< DPM++ 2M Karras (recommended) */
33
+ RAC_DIFFUSION_SCHEDULER_DPM_PP_2M = 1, /**< DPM++ 2M */
34
+ RAC_DIFFUSION_SCHEDULER_DPM_PP_2M_SDE = 2, /**< DPM++ 2M SDE */
35
+ RAC_DIFFUSION_SCHEDULER_DDIM = 3, /**< DDIM */
36
+ RAC_DIFFUSION_SCHEDULER_EULER = 4, /**< Euler */
37
+ RAC_DIFFUSION_SCHEDULER_EULER_ANCESTRAL = 5, /**< Euler Ancestral */
38
+ RAC_DIFFUSION_SCHEDULER_PNDM = 6, /**< PNDM */
39
+ RAC_DIFFUSION_SCHEDULER_LMS = 7, /**< LMS */
40
+ } rac_diffusion_scheduler_t;
41
+
42
+ /**
43
+ * @brief Model variant types
44
+ *
45
+ * Different Stable Diffusion model variants with different capabilities.
46
+ */
47
+ typedef enum rac_diffusion_model_variant {
48
+ RAC_DIFFUSION_MODEL_SD_1_5 = 0, /**< Stable Diffusion 1.5 (512x512 default) */
49
+ RAC_DIFFUSION_MODEL_SD_2_1 = 1, /**< Stable Diffusion 2.1 (768x768 default) */
50
+ RAC_DIFFUSION_MODEL_SDXL = 2, /**< SDXL (1024x1024 default, requires 8GB+ RAM) */
51
+ RAC_DIFFUSION_MODEL_SDXL_TURBO = 3, /**< SDXL Turbo (fast, fewer steps, no CFG) */
52
+ RAC_DIFFUSION_MODEL_SDXS = 4, /**< SDXS - Ultra-fast 1-step model (no CFG) */
53
+ RAC_DIFFUSION_MODEL_LCM = 5, /**< LCM - Latent Consistency Model (4 steps) */
54
+ } rac_diffusion_model_variant_t;
55
+
56
+ /**
57
+ * @brief Generation mode
58
+ */
59
+ typedef enum rac_diffusion_mode {
60
+ RAC_DIFFUSION_MODE_TEXT_TO_IMAGE = 0, /**< Generate image from text prompt */
61
+ RAC_DIFFUSION_MODE_IMAGE_TO_IMAGE = 1, /**< Transform input image with prompt */
62
+ RAC_DIFFUSION_MODE_INPAINTING = 2, /**< Edit specific regions with mask */
63
+ } rac_diffusion_mode_t;
64
+
65
+ // =============================================================================
66
+ // TOKENIZER CONFIGURATION
67
+ // =============================================================================
68
+
69
+ /**
70
+ * @brief Tokenizer source presets
71
+ *
72
+ * Predefined HuggingFace repository sources for tokenizer files.
73
+ * Apple's compiled CoreML models don't include tokenizer files (vocab.json, merges.txt),
74
+ * so they must be downloaded separately from HuggingFace.
75
+ *
76
+ * Developers can use RAC_DIFFUSION_TOKENIZER_CUSTOM with a custom_base_url
77
+ * to specify their own tokenizer source.
78
+ */
79
+ typedef enum rac_diffusion_tokenizer_source {
80
+ /** Stable Diffusion 1.x tokenizer (CLIP ViT-L/14)
81
+ * Source: runwayml/stable-diffusion-v1-5 */
82
+ RAC_DIFFUSION_TOKENIZER_SD_1_5 = 0,
83
+
84
+ /** Stable Diffusion 2.x tokenizer (OpenCLIP ViT-H/14)
85
+ * Source: stabilityai/stable-diffusion-2-1 */
86
+ RAC_DIFFUSION_TOKENIZER_SD_2_X = 1,
87
+
88
+ /** Stable Diffusion XL tokenizer (dual tokenizers)
89
+ * Source: stabilityai/stable-diffusion-xl-base-1.0 */
90
+ RAC_DIFFUSION_TOKENIZER_SDXL = 2,
91
+
92
+ /** Custom tokenizer from a developer-specified URL
93
+ * Requires custom_base_url to be set in rac_diffusion_tokenizer_config_t */
94
+ RAC_DIFFUSION_TOKENIZER_CUSTOM = 99,
95
+ } rac_diffusion_tokenizer_source_t;
96
+
97
+ /**
98
+ * @brief Tokenizer configuration
99
+ *
100
+ * Configuration for downloading and using tokenizer files.
101
+ * The SDK will automatically download missing tokenizer files (vocab.json, merges.txt)
102
+ * from the specified source URL.
103
+ *
104
+ * Example for custom URL:
105
+ * @code
106
+ * rac_diffusion_tokenizer_config_t tokenizer_config = {
107
+ * .source = RAC_DIFFUSION_TOKENIZER_CUSTOM,
108
+ * .custom_base_url = "https://huggingface.co/my-org/my-model/resolve/main/tokenizer",
109
+ * .auto_download = RAC_TRUE
110
+ * };
111
+ * @endcode
112
+ */
113
+ typedef struct rac_diffusion_tokenizer_config {
114
+ /** Tokenizer source preset (SD15, SD21, SDXL, or CUSTOM) */
115
+ rac_diffusion_tokenizer_source_t source;
116
+
117
+ /** Custom base URL for tokenizer files (only used when source == CUSTOM)
118
+ * Should be a URL directory containing vocab.json and merges.txt
119
+ * Example: "https://huggingface.co/my-org/my-model/resolve/main/tokenizer"
120
+ * The SDK will append "/vocab.json" and "/merges.txt" to download files */
121
+ const char* custom_base_url;
122
+
123
+ /** Automatically download missing tokenizer files (default: true) */
124
+ rac_bool_t auto_download;
125
+ } rac_diffusion_tokenizer_config_t;
126
+
127
+ /**
128
+ * @brief Default tokenizer configuration
129
+ */
130
+ static const rac_diffusion_tokenizer_config_t RAC_DIFFUSION_TOKENIZER_CONFIG_DEFAULT = {
131
+ .source = RAC_DIFFUSION_TOKENIZER_SD_1_5,
132
+ .custom_base_url = RAC_NULL,
133
+ .auto_download = RAC_TRUE};
134
+
135
+ // =============================================================================
136
+ // CONFIGURATION - Component configuration
137
+ // =============================================================================
138
+
139
+ /**
140
+ * @brief Diffusion component configuration
141
+ *
142
+ * Configuration for initializing the diffusion component.
143
+ */
144
+ typedef struct rac_diffusion_config {
145
+ /** Model ID (optional - uses default if NULL) */
146
+ const char* model_id;
147
+
148
+ /** Preferred framework (use RAC_FRAMEWORK_UNKNOWN for auto) */
149
+ int32_t preferred_framework;
150
+
151
+ /** Model variant (SD 1.5, SD 2.1, SDXL, etc.) */
152
+ rac_diffusion_model_variant_t model_variant;
153
+
154
+ /** Enable safety checker for NSFW content filtering (default: true) */
155
+ rac_bool_t enable_safety_checker;
156
+
157
+ /** Reduce memory footprint (may reduce quality, default: false) */
158
+ rac_bool_t reduce_memory;
159
+
160
+ /** Tokenizer configuration for downloading missing tokenizer files
161
+ * Apple's compiled CoreML models don't include tokenizer files */
162
+ rac_diffusion_tokenizer_config_t tokenizer;
163
+ } rac_diffusion_config_t;
164
+
165
+ /**
166
+ * @brief Default diffusion configuration
167
+ */
168
+ static const rac_diffusion_config_t RAC_DIFFUSION_CONFIG_DEFAULT = {
169
+ .model_id = RAC_NULL,
170
+ .preferred_framework = 99, // RAC_FRAMEWORK_UNKNOWN
171
+ .model_variant = RAC_DIFFUSION_MODEL_SD_1_5,
172
+ .enable_safety_checker = RAC_TRUE,
173
+ .reduce_memory = RAC_FALSE,
174
+ .tokenizer = {.source = RAC_DIFFUSION_TOKENIZER_SD_1_5,
175
+ .custom_base_url = RAC_NULL,
176
+ .auto_download = RAC_TRUE}};
177
+
178
+ // =============================================================================
179
+ // OPTIONS - Generation options
180
+ // =============================================================================
181
+
182
+ /**
183
+ * @brief Diffusion generation options
184
+ *
185
+ * Options for controlling image generation.
186
+ */
187
+ typedef struct rac_diffusion_options {
188
+ /** Text prompt describing the desired image */
189
+ const char* prompt;
190
+
191
+ /** Negative prompt - things to avoid in the image (can be NULL) */
192
+ const char* negative_prompt;
193
+
194
+ /** Output image width in pixels (default: 512 for SD 1.5, 1024 for SDXL) */
195
+ int32_t width;
196
+
197
+ /** Output image height in pixels (default: 512 for SD 1.5, 1024 for SDXL) */
198
+ int32_t height;
199
+
200
+ /** Number of denoising steps (default: 28, range: 10-50) */
201
+ int32_t steps;
202
+
203
+ /** Classifier-free guidance scale (default: 7.5, range: 1.0-20.0) */
204
+ float guidance_scale;
205
+
206
+ /** Random seed for reproducibility (-1 for random, default: -1) */
207
+ int64_t seed;
208
+
209
+ /** Scheduler/sampler algorithm (default: DPM++ 2M Karras) */
210
+ rac_diffusion_scheduler_t scheduler;
211
+
212
+ // --- Image-to-image / Inpainting options ---
213
+
214
+ /** Generation mode (text-to-image, img2img, inpainting) */
215
+ rac_diffusion_mode_t mode;
216
+
217
+ /** Input image RGBA data for img2img/inpainting (can be NULL) */
218
+ const uint8_t* input_image_data;
219
+
220
+ /** Input image data size in bytes */
221
+ size_t input_image_size;
222
+
223
+ /** Input image width (required if input_image_data is set) */
224
+ int32_t input_image_width;
225
+
226
+ /** Input image height (required if input_image_data is set) */
227
+ int32_t input_image_height;
228
+
229
+ /** Mask image data for inpainting - grayscale (can be NULL) */
230
+ const uint8_t* mask_data;
231
+
232
+ /** Mask data size in bytes */
233
+ size_t mask_size;
234
+
235
+ /** Denoising strength for img2img (0.0-1.0, default: 0.75) */
236
+ float denoise_strength;
237
+
238
+ // --- Progress reporting options ---
239
+
240
+ /** Report intermediate images during generation (default: false) */
241
+ rac_bool_t report_intermediate_images;
242
+
243
+ /** Report progress every N steps (default: 1) */
244
+ int32_t progress_stride;
245
+ } rac_diffusion_options_t;
246
+
247
+ /**
248
+ * @brief Default diffusion generation options
249
+ */
250
+ static const rac_diffusion_options_t RAC_DIFFUSION_OPTIONS_DEFAULT = {
251
+ .prompt = RAC_NULL,
252
+ .negative_prompt = RAC_NULL,
253
+ .width = 512,
254
+ .height = 512,
255
+ .steps = 28,
256
+ .guidance_scale = 7.5f,
257
+ .seed = -1,
258
+ .scheduler = RAC_DIFFUSION_SCHEDULER_DPM_PP_2M_KARRAS,
259
+ .mode = RAC_DIFFUSION_MODE_TEXT_TO_IMAGE,
260
+ .input_image_data = RAC_NULL,
261
+ .input_image_size = 0,
262
+ .input_image_width = 0,
263
+ .input_image_height = 0,
264
+ .mask_data = RAC_NULL,
265
+ .mask_size = 0,
266
+ .denoise_strength = 0.75f,
267
+ .report_intermediate_images = RAC_FALSE,
268
+ .progress_stride = 1};
269
+
270
+ // =============================================================================
271
+ // PROGRESS - Generation progress
272
+ // =============================================================================
273
+
274
+ /**
275
+ * @brief Diffusion generation progress
276
+ *
277
+ * Reports progress during image generation.
278
+ */
279
+ typedef struct rac_diffusion_progress {
280
+ /** Progress percentage (0.0 - 1.0) */
281
+ float progress;
282
+
283
+ /** Current step number (1-based) */
284
+ int32_t current_step;
285
+
286
+ /** Total number of steps */
287
+ int32_t total_steps;
288
+
289
+ /** Current stage description (e.g., "Encoding", "Denoising", "Decoding") */
290
+ const char* stage;
291
+
292
+ /** Intermediate image RGBA data (can be NULL if not requested) */
293
+ const uint8_t* intermediate_image_data;
294
+
295
+ /** Intermediate image data size */
296
+ size_t intermediate_image_size;
297
+
298
+ /** Intermediate image width */
299
+ int32_t intermediate_image_width;
300
+
301
+ /** Intermediate image height */
302
+ int32_t intermediate_image_height;
303
+ } rac_diffusion_progress_t;
304
+
305
+ // =============================================================================
306
+ // RESULT - Generation result
307
+ // =============================================================================
308
+
309
+ /**
310
+ * @brief Diffusion generation result
311
+ *
312
+ * Contains the generated image and metadata.
313
+ */
314
+ typedef struct rac_diffusion_result {
315
+ /** Generated image RGBA data (owned, must be freed with rac_diffusion_result_free) */
316
+ uint8_t* image_data;
317
+
318
+ /** Image data size in bytes */
319
+ size_t image_size;
320
+
321
+ /** Image width in pixels */
322
+ int32_t width;
323
+
324
+ /** Image height in pixels */
325
+ int32_t height;
326
+
327
+ /** Seed used for generation (useful for reproducibility) */
328
+ int64_t seed_used;
329
+
330
+ /** Total generation time in milliseconds */
331
+ int64_t generation_time_ms;
332
+
333
+ /** Whether the image was flagged by safety checker */
334
+ rac_bool_t safety_flagged;
335
+
336
+ /** Error code if generation failed (RAC_SUCCESS on success) */
337
+ rac_result_t error_code;
338
+
339
+ /** Error message if generation failed (can be NULL) */
340
+ char* error_message;
341
+ } rac_diffusion_result_t;
342
+
343
+ // =============================================================================
344
+ // INFO - Service information
345
+ // =============================================================================
346
+
347
+ /**
348
+ * @brief Diffusion service information
349
+ *
350
+ * Information about the loaded diffusion service.
351
+ */
352
+ typedef struct rac_diffusion_info {
353
+ /** Whether the service is ready for generation */
354
+ rac_bool_t is_ready;
355
+
356
+ /** Current model identifier (can be NULL) */
357
+ const char* current_model;
358
+
359
+ /** Model variant */
360
+ rac_diffusion_model_variant_t model_variant;
361
+
362
+ /** Whether text-to-image is supported */
363
+ rac_bool_t supports_text_to_image;
364
+
365
+ /** Whether image-to-image is supported */
366
+ rac_bool_t supports_image_to_image;
367
+
368
+ /** Whether inpainting is supported */
369
+ rac_bool_t supports_inpainting;
370
+
371
+ /** Whether safety checker is enabled */
372
+ rac_bool_t safety_checker_enabled;
373
+
374
+ /** Maximum supported width */
375
+ int32_t max_width;
376
+
377
+ /** Maximum supported height */
378
+ int32_t max_height;
379
+ } rac_diffusion_info_t;
380
+
381
+ // =============================================================================
382
+ // CALLBACKS
383
+ // =============================================================================
384
+
385
+ /**
386
+ * @brief Diffusion progress callback
387
+ *
388
+ * Called during generation to report progress.
389
+ *
390
+ * @param progress Progress information
391
+ * @param user_data User-provided context
392
+ * @return RAC_TRUE to continue, RAC_FALSE to cancel generation
393
+ */
394
+ typedef rac_bool_t (*rac_diffusion_progress_callback_fn)(const rac_diffusion_progress_t* progress,
395
+ void* user_data);
396
+
397
+ /**
398
+ * @brief Diffusion completion callback
399
+ *
400
+ * Called when generation completes successfully.
401
+ *
402
+ * @param result Generation result
403
+ * @param user_data User-provided context
404
+ */
405
+ typedef void (*rac_diffusion_complete_callback_fn)(const rac_diffusion_result_t* result,
406
+ void* user_data);
407
+
408
+ /**
409
+ * @brief Diffusion error callback
410
+ *
411
+ * Called when generation fails.
412
+ *
413
+ * @param error_code Error code
414
+ * @param error_message Error message
415
+ * @param user_data User-provided context
416
+ */
417
+ typedef void (*rac_diffusion_error_callback_fn)(rac_result_t error_code, const char* error_message,
418
+ void* user_data);
419
+
420
+ // =============================================================================
421
+ // CAPABILITY FLAGS
422
+ // =============================================================================
423
+
424
+ /** Supports text-to-image generation */
425
+ #define RAC_DIFFUSION_CAP_TEXT_TO_IMAGE (1 << 0)
426
+
427
+ /** Supports image-to-image transformation */
428
+ #define RAC_DIFFUSION_CAP_IMAGE_TO_IMAGE (1 << 1)
429
+
430
+ /** Supports inpainting with mask */
431
+ #define RAC_DIFFUSION_CAP_INPAINTING (1 << 2)
432
+
433
+ /** Supports intermediate image reporting */
434
+ #define RAC_DIFFUSION_CAP_INTERMEDIATE_IMAGES (1 << 3)
435
+
436
+ /** Has safety checker */
437
+ #define RAC_DIFFUSION_CAP_SAFETY_CHECKER (1 << 4)
438
+
439
+ // =============================================================================
440
+ // MEMORY MANAGEMENT
441
+ // =============================================================================
442
+
443
+ /**
444
+ * @brief Free diffusion result resources
445
+ *
446
+ * @param result Result to free (can be NULL)
447
+ */
448
+ RAC_API void rac_diffusion_result_free(rac_diffusion_result_t* result);
449
+
450
+ #ifdef __cplusplus
451
+ }
452
+ #endif
453
+
454
+ #endif /* RAC_DIFFUSION_TYPES_H */
@@ -62,24 +62,10 @@ const char* rac_endpoint_device_registration(rac_environment_t env);
62
62
  const char* rac_endpoint_telemetry(rac_environment_t env);
63
63
 
64
64
  /**
65
- * @brief Build model assignments endpoint with query params
66
- * @param device_type Device type (e.g., "iphone", "android_phone")
67
- * @param platform Platform (e.g., "ios", "android")
68
- * @param out_buffer Buffer to write endpoint path
69
- * @param buffer_size Size of buffer
70
- * @return Length of written string, or -1 on error
71
- */
72
- int rac_endpoint_model_assignments(const char* device_type, const char* platform, char* out_buffer,
73
- size_t buffer_size);
74
-
75
- /**
76
- * @brief Build model download endpoint
77
- * @param model_id The model ID
78
- * @param out_buffer Buffer to write endpoint path
79
- * @param buffer_size Size of buffer
80
- * @return Length of written string, or -1 on error
65
+ * @brief Get model assignments endpoint
66
+ * @return Endpoint path string
81
67
  */
82
- int rac_endpoint_model_download(const char* model_id, char* out_buffer, size_t buffer_size);
68
+ const char* rac_endpoint_model_assignments(void);
83
69
 
84
70
  // =============================================================================
85
71
  // Full URL Building
@@ -0,0 +1,215 @@
1
+ /**
2
+ * @file rac_image_utils.h
3
+ * @brief RunAnywhere Commons - Image Utilities
4
+ *
5
+ * Image loading and processing utilities for VLM backends.
6
+ * Supports loading from file paths, decoding base64, and resizing.
7
+ */
8
+
9
+ #ifndef RAC_IMAGE_UTILS_H
10
+ #define RAC_IMAGE_UTILS_H
11
+
12
+ #include <RACommons/rac_error.h>
13
+ #include <RACommons/rac_types.h>
14
+
15
+ #ifdef __cplusplus
16
+ extern "C" {
17
+ #endif
18
+
19
+ // =============================================================================
20
+ // IMAGE DATA STRUCTURES
21
+ // =============================================================================
22
+
23
+ /**
24
+ * @brief Loaded image data
25
+ *
26
+ * Contains RGB pixel data after loading an image.
27
+ * Must be freed with rac_image_free().
28
+ */
29
+ typedef struct rac_image_data {
30
+ /** Raw RGB pixel data (RGBRGBRGB...) */
31
+ uint8_t* pixels;
32
+
33
+ /** Image width in pixels */
34
+ int32_t width;
35
+
36
+ /** Image height in pixels */
37
+ int32_t height;
38
+
39
+ /** Number of channels (3 for RGB) */
40
+ int32_t channels;
41
+
42
+ /** Total size in bytes (width * height * channels) */
43
+ size_t size;
44
+ } rac_image_data_t;
45
+
46
+ /**
47
+ * @brief Normalized float image data
48
+ *
49
+ * Contains normalized float32 pixel data (values in [-1, 1] or [0, 1]).
50
+ * Used by vision encoders.
51
+ */
52
+ typedef struct rac_image_float {
53
+ /** Normalized float pixel data */
54
+ float* pixels;
55
+
56
+ /** Image width in pixels */
57
+ int32_t width;
58
+
59
+ /** Image height in pixels */
60
+ int32_t height;
61
+
62
+ /** Number of channels (3 for RGB) */
63
+ int32_t channels;
64
+
65
+ /** Total number of floats (width * height * channels) */
66
+ size_t count;
67
+ } rac_image_float_t;
68
+
69
+ // =============================================================================
70
+ // IMAGE LOADING
71
+ // =============================================================================
72
+
73
+ /**
74
+ * @brief Load an image from a file path
75
+ *
76
+ * Supports JPEG, PNG, BMP, GIF, and other common formats via stb_image.
77
+ * Output is always RGB (3 channels).
78
+ *
79
+ * @param file_path Path to the image file
80
+ * @param out_image Output: Loaded image data (must be freed with rac_image_free)
81
+ * @return RAC_SUCCESS or error code
82
+ */
83
+ RAC_API rac_result_t rac_image_load_file(const char* file_path, rac_image_data_t* out_image);
84
+
85
+ /**
86
+ * @brief Decode a base64-encoded image
87
+ *
88
+ * Decodes base64 data and loads the image.
89
+ * Supports the same formats as rac_image_load_file.
90
+ *
91
+ * @param base64_data Base64-encoded image data
92
+ * @param data_size Length of the base64 string
93
+ * @param out_image Output: Loaded image data (must be freed with rac_image_free)
94
+ * @return RAC_SUCCESS or error code
95
+ */
96
+ RAC_API rac_result_t rac_image_decode_base64(const char* base64_data, size_t data_size,
97
+ rac_image_data_t* out_image);
98
+
99
+ /**
100
+ * @brief Decode image from raw bytes
101
+ *
102
+ * Decodes an image from raw bytes (e.g., from network response).
103
+ *
104
+ * @param data Raw image data (JPEG, PNG, etc.)
105
+ * @param data_size Size of the data in bytes
106
+ * @param out_image Output: Loaded image data (must be freed with rac_image_free)
107
+ * @return RAC_SUCCESS or error code
108
+ */
109
+ RAC_API rac_result_t rac_image_decode_bytes(const uint8_t* data, size_t data_size,
110
+ rac_image_data_t* out_image);
111
+
112
+ // =============================================================================
113
+ // IMAGE PROCESSING
114
+ // =============================================================================
115
+
116
+ /**
117
+ * @brief Resize an image
118
+ *
119
+ * Resizes the image to the specified dimensions using bilinear interpolation.
120
+ *
121
+ * @param image Input image
122
+ * @param new_width Target width
123
+ * @param new_height Target height
124
+ * @param out_image Output: Resized image (must be freed with rac_image_free)
125
+ * @return RAC_SUCCESS or error code
126
+ */
127
+ RAC_API rac_result_t rac_image_resize(const rac_image_data_t* image, int32_t new_width,
128
+ int32_t new_height, rac_image_data_t* out_image);
129
+
130
+ /**
131
+ * @brief Resize an image maintaining aspect ratio
132
+ *
133
+ * Resizes the image so that the longest dimension equals max_size.
134
+ * Aspect ratio is preserved.
135
+ *
136
+ * @param image Input image
137
+ * @param max_size Maximum dimension (width or height)
138
+ * @param out_image Output: Resized image (must be freed with rac_image_free)
139
+ * @return RAC_SUCCESS or error code
140
+ */
141
+ RAC_API rac_result_t rac_image_resize_max(const rac_image_data_t* image, int32_t max_size,
142
+ rac_image_data_t* out_image);
143
+
144
+ /**
145
+ * @brief Normalize image to float values
146
+ *
147
+ * Converts uint8 pixels to float32 with optional mean/std normalization.
148
+ * Commonly used for vision encoders (CLIP, SigLIP, etc.).
149
+ *
150
+ * Formula: pixel_normalized = (pixel / 255.0 - mean) / std
151
+ *
152
+ * @param image Input image
153
+ * @param mean Per-channel mean values (array of 3 floats, or NULL for [0,0,0])
154
+ * @param std Per-channel std values (array of 3 floats, or NULL for [1,1,1])
155
+ * @param out_float Output: Normalized float image (must be freed with rac_image_float_free)
156
+ * @return RAC_SUCCESS or error code
157
+ */
158
+ RAC_API rac_result_t rac_image_normalize(const rac_image_data_t* image, const float* mean,
159
+ const float* std, rac_image_float_t* out_float);
160
+
161
+ /**
162
+ * @brief Convert RGB to CHW format
163
+ *
164
+ * Converts from HWC (Height, Width, Channels) to CHW format.
165
+ * Many neural networks expect CHW format.
166
+ *
167
+ * @param image Input float image in HWC format
168
+ * @param out_chw Output: Float image in CHW format (must be freed with rac_image_float_free)
169
+ * @return RAC_SUCCESS or error code
170
+ */
171
+ RAC_API rac_result_t rac_image_to_chw(const rac_image_float_t* image, rac_image_float_t* out_chw);
172
+
173
+ // =============================================================================
174
+ // MEMORY MANAGEMENT
175
+ // =============================================================================
176
+
177
+ /**
178
+ * @brief Free image data
179
+ *
180
+ * Frees the pixel data allocated by image loading functions.
181
+ *
182
+ * @param image Image to free (can be NULL)
183
+ */
184
+ RAC_API void rac_image_free(rac_image_data_t* image);
185
+
186
+ /**
187
+ * @brief Free float image data
188
+ *
189
+ * Frees the pixel data allocated by normalization functions.
190
+ *
191
+ * @param image Float image to free (can be NULL)
192
+ */
193
+ RAC_API void rac_image_float_free(rac_image_float_t* image);
194
+
195
+ // =============================================================================
196
+ // UTILITY FUNCTIONS
197
+ // =============================================================================
198
+
199
+ /**
200
+ * @brief Calculate resized dimensions maintaining aspect ratio
201
+ *
202
+ * @param width Original width
203
+ * @param height Original height
204
+ * @param max_size Maximum dimension
205
+ * @param out_width Output: New width
206
+ * @param out_height Output: New height
207
+ */
208
+ RAC_API void rac_image_calc_resize(int32_t width, int32_t height, int32_t max_size,
209
+ int32_t* out_width, int32_t* out_height);
210
+
211
+ #ifdef __cplusplus
212
+ }
213
+ #endif
214
+
215
+ #endif /* RAC_IMAGE_UTILS_H */