@runanywhere/core 0.17.7 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +218 -2
- package/RunAnywhereCore.podspec +1 -0
- package/android/CMakeLists.txt +24 -2
- package/android/build.gradle +61 -9
- package/android/src/main/cpp/cpp-adapter.cpp +51 -3
- package/android/src/main/include/rac/backends/rac_vlm_llamacpp.h +216 -0
- package/android/src/main/include/rac/core/capabilities/rac_lifecycle.h +3 -1
- package/android/src/main/include/rac/core/rac_core.h +11 -0
- package/android/src/main/include/rac/core/rac_types.h +8 -6
- package/android/src/main/include/rac/features/diffusion/rac_diffusion.h +22 -0
- package/android/src/main/include/rac/features/diffusion/rac_diffusion_component.h +263 -0
- package/android/src/main/include/rac/features/diffusion/rac_diffusion_model_registry.h +358 -0
- package/android/src/main/include/rac/features/diffusion/rac_diffusion_service.h +187 -0
- package/android/src/main/include/rac/features/diffusion/rac_diffusion_tokenizer.h +167 -0
- package/android/src/main/include/rac/features/diffusion/rac_diffusion_types.h +454 -0
- package/android/src/main/include/rac/features/llm/rac_tool_calling.h +373 -0
- package/android/src/main/include/rac/features/platform/rac_diffusion_platform.h +305 -0
- package/android/src/main/include/rac/features/vad/rac_vad_energy.h +1 -1
- package/android/src/main/include/rac/features/vlm/rac_vlm.h +16 -0
- package/android/src/main/include/rac/features/vlm/rac_vlm_component.h +168 -0
- package/android/src/main/include/rac/features/vlm/rac_vlm_service.h +206 -0
- package/android/src/main/include/rac/features/vlm/rac_vlm_types.h +417 -0
- package/android/src/main/include/rac/infrastructure/model_management/rac_model_registry.h +15 -0
- package/android/src/main/include/rac/infrastructure/model_management/rac_model_types.h +3 -0
- package/android/src/main/include/rac/utils/rac_image_utils.h +215 -0
- package/android/src/main/java/com/margelo/nitro/runanywhere/PlatformAdapterBridge.kt +201 -1
- package/android/src/main/jniLibs/arm64-v8a/libc++_shared.so +0 -0
- package/android/src/main/jniLibs/arm64-v8a/libomp.so +0 -0
- package/android/src/main/jniLibs/arm64-v8a/librac_commons.so +0 -0
- package/android/src/main/jniLibs/arm64-v8a/librunanywhere_jni.so +0 -0
- package/android/src/main/jniLibs/x86_64/libc++_shared.so +0 -0
- package/android/src/main/jniLibs/x86_64/libomp.so +0 -0
- package/android/src/main/jniLibs/x86_64/librac_commons.so +0 -0
- package/android/src/main/jniLibs/x86_64/librunanywhere_jni.so +0 -0
- package/cpp/HybridRunAnywhereCore.cpp +259 -160
- package/cpp/HybridRunAnywhereCore.hpp +11 -0
- package/cpp/bridges/InitBridge.cpp +234 -3
- package/cpp/bridges/PlatformDownloadBridge.h +44 -0
- package/cpp/bridges/ToolCallingBridge.cpp +188 -0
- package/cpp/bridges/ToolCallingBridge.hpp +98 -0
- package/cpp/third_party/nlohmann/json.hpp +24765 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/RACommons.h +18 -4
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_core.h +11 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_diffusion.h +22 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_diffusion_component.h +263 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_diffusion_model_registry.h +358 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_diffusion_platform.h +305 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_diffusion_service.h +187 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_diffusion_tokenizer.h +167 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_diffusion_types.h +454 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_endpoints.h +3 -17
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_image_utils.h +215 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_lifecycle.h +3 -1
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_model_assignment.h +4 -20
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_model_registry.h +15 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_model_types.h +3 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_tool_calling.h +373 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_types.h +8 -6
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_vad_energy.h +1 -1
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_vlm.h +16 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_vlm_component.h +168 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_vlm_llamacpp.h +216 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_vlm_service.h +206 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/Headers/rac_vlm_types.h +417 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64/RACommons.framework/RACommons +0 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/RACommons.h +18 -4
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_core.h +11 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_diffusion.h +22 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_diffusion_component.h +263 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_diffusion_model_registry.h +358 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_diffusion_platform.h +305 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_diffusion_service.h +187 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_diffusion_tokenizer.h +167 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_diffusion_types.h +454 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_endpoints.h +3 -17
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_image_utils.h +215 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_lifecycle.h +3 -1
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_model_assignment.h +4 -20
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_model_registry.h +15 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_model_types.h +3 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_tool_calling.h +373 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_types.h +8 -6
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_vad_energy.h +1 -1
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_vlm.h +16 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_vlm_component.h +168 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_vlm_llamacpp.h +216 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_vlm_service.h +206 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/Headers/rac_vlm_types.h +417 -0
- package/ios/Binaries/RACommons.xcframework/ios-arm64_x86_64-simulator/RACommons.framework/RACommons +0 -0
- package/ios/PlatformAdapterBridge.h +24 -1
- package/ios/PlatformAdapterBridge.m +243 -0
- package/nitrogen/generated/shared/c++/HybridRunAnywhereCoreSpec.cpp +4 -0
- package/nitrogen/generated/shared/c++/HybridRunAnywhereCoreSpec.hpp +4 -0
- package/package.json +8 -4
- package/src/Foundation/Security/SecureStorageService.ts +12 -6
- package/src/Public/Extensions/RunAnywhere+Models.ts +5 -3
- package/src/Public/Extensions/RunAnywhere+STT.ts +7 -2
- package/src/Public/Extensions/RunAnywhere+ToolCalling.ts +472 -0
- package/src/Public/Extensions/index.ts +16 -0
- package/src/Public/RunAnywhere.ts +18 -0
- package/src/index.ts +0 -1
- package/src/services/Network/index.ts +0 -1
- package/src/services/index.ts +0 -1
- package/src/specs/RunAnywhereCore.nitro.ts +72 -0
- package/src/types/ToolCallingTypes.ts +198 -0
- package/src/types/index.ts +13 -0
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file rac_diffusion_service.h
|
|
3
|
+
* @brief RunAnywhere Commons - Diffusion Service Interface
|
|
4
|
+
*
|
|
5
|
+
* Defines the generic diffusion service API and vtable for multi-backend dispatch.
|
|
6
|
+
* Backends (CoreML, ONNX, Platform) implement the vtable and register
|
|
7
|
+
* with the service registry.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
#ifndef RAC_DIFFUSION_SERVICE_H
|
|
11
|
+
#define RAC_DIFFUSION_SERVICE_H
|
|
12
|
+
|
|
13
|
+
#include <RACommons/rac_error.h>
|
|
14
|
+
#include <RACommons/rac_diffusion_types.h>
|
|
15
|
+
|
|
16
|
+
#ifdef __cplusplus
|
|
17
|
+
extern "C" {
|
|
18
|
+
#endif
|
|
19
|
+
|
|
20
|
+
// =============================================================================
|
|
21
|
+
// SERVICE VTABLE - Backend implementations provide this
|
|
22
|
+
// =============================================================================
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Diffusion Service operations vtable.
|
|
26
|
+
* Each backend implements these functions and provides a static vtable.
|
|
27
|
+
*/
|
|
28
|
+
typedef struct rac_diffusion_service_ops {
|
|
29
|
+
/** Initialize the service with a model path */
|
|
30
|
+
rac_result_t (*initialize)(void* impl, const char* model_path,
|
|
31
|
+
const rac_diffusion_config_t* config);
|
|
32
|
+
|
|
33
|
+
/** Generate image (blocking) */
|
|
34
|
+
rac_result_t (*generate)(void* impl, const rac_diffusion_options_t* options,
|
|
35
|
+
rac_diffusion_result_t* out_result);
|
|
36
|
+
|
|
37
|
+
/** Generate image with progress callback */
|
|
38
|
+
rac_result_t (*generate_with_progress)(void* impl, const rac_diffusion_options_t* options,
|
|
39
|
+
rac_diffusion_progress_callback_fn progress_callback,
|
|
40
|
+
void* user_data, rac_diffusion_result_t* out_result);
|
|
41
|
+
|
|
42
|
+
/** Get service info */
|
|
43
|
+
rac_result_t (*get_info)(void* impl, rac_diffusion_info_t* out_info);
|
|
44
|
+
|
|
45
|
+
/** Get supported capabilities as bitmask */
|
|
46
|
+
uint32_t (*get_capabilities)(void* impl);
|
|
47
|
+
|
|
48
|
+
/** Cancel ongoing generation */
|
|
49
|
+
rac_result_t (*cancel)(void* impl);
|
|
50
|
+
|
|
51
|
+
/** Cleanup/unload model (keeps service alive) */
|
|
52
|
+
rac_result_t (*cleanup)(void* impl);
|
|
53
|
+
|
|
54
|
+
/** Destroy the service */
|
|
55
|
+
void (*destroy)(void* impl);
|
|
56
|
+
} rac_diffusion_service_ops_t;
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Diffusion Service instance.
|
|
60
|
+
* Contains vtable pointer and backend-specific implementation.
|
|
61
|
+
*/
|
|
62
|
+
typedef struct rac_diffusion_service {
|
|
63
|
+
/** Vtable with backend operations */
|
|
64
|
+
const rac_diffusion_service_ops_t* ops;
|
|
65
|
+
|
|
66
|
+
/** Backend-specific implementation handle */
|
|
67
|
+
void* impl;
|
|
68
|
+
|
|
69
|
+
/** Model ID for reference */
|
|
70
|
+
const char* model_id;
|
|
71
|
+
} rac_diffusion_service_t;
|
|
72
|
+
|
|
73
|
+
// =============================================================================
|
|
74
|
+
// PUBLIC API - Generic service functions
|
|
75
|
+
// =============================================================================
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* @brief Create a diffusion service
|
|
79
|
+
*
|
|
80
|
+
* Routes through service registry to find appropriate backend.
|
|
81
|
+
*
|
|
82
|
+
* @param model_id Model identifier (registry ID or path to model)
|
|
83
|
+
* @param out_handle Output: Handle to the created service
|
|
84
|
+
* @return RAC_SUCCESS or error code
|
|
85
|
+
*/
|
|
86
|
+
RAC_API rac_result_t rac_diffusion_create(const char* model_id, rac_handle_t* out_handle);
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* @brief Create a diffusion service with configuration
|
|
90
|
+
*
|
|
91
|
+
* Routes through service registry to find appropriate backend, honoring
|
|
92
|
+
* configuration hints such as preferred framework when provided.
|
|
93
|
+
*
|
|
94
|
+
* @param model_id Model identifier (registry ID or path to model)
|
|
95
|
+
* @param config Optional configuration (can be NULL)
|
|
96
|
+
* @param out_handle Output: Handle to the created service
|
|
97
|
+
* @return RAC_SUCCESS or error code
|
|
98
|
+
*/
|
|
99
|
+
RAC_API rac_result_t rac_diffusion_create_with_config(const char* model_id,
|
|
100
|
+
const rac_diffusion_config_t* config,
|
|
101
|
+
rac_handle_t* out_handle);
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* @brief Initialize a diffusion service
|
|
105
|
+
*
|
|
106
|
+
* @param handle Service handle
|
|
107
|
+
* @param model_path Path to the model directory
|
|
108
|
+
* @param config Configuration (can be NULL for defaults)
|
|
109
|
+
* @return RAC_SUCCESS or error code
|
|
110
|
+
*/
|
|
111
|
+
RAC_API rac_result_t rac_diffusion_initialize(rac_handle_t handle, const char* model_path,
|
|
112
|
+
const rac_diffusion_config_t* config);
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* @brief Generate an image from prompt
|
|
116
|
+
*
|
|
117
|
+
* Blocking call that generates an image.
|
|
118
|
+
*
|
|
119
|
+
* @param handle Service handle
|
|
120
|
+
* @param options Generation options
|
|
121
|
+
* @param out_result Output: Generation result (caller must free with rac_diffusion_result_free)
|
|
122
|
+
* @return RAC_SUCCESS or error code
|
|
123
|
+
*/
|
|
124
|
+
RAC_API rac_result_t rac_diffusion_generate(rac_handle_t handle,
|
|
125
|
+
const rac_diffusion_options_t* options,
|
|
126
|
+
rac_diffusion_result_t* out_result);
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* @brief Generate an image with progress reporting
|
|
130
|
+
*
|
|
131
|
+
* @param handle Service handle
|
|
132
|
+
* @param options Generation options
|
|
133
|
+
* @param progress_callback Callback for progress updates
|
|
134
|
+
* @param user_data User context passed to callback
|
|
135
|
+
* @param out_result Output: Generation result (caller must free with rac_diffusion_result_free)
|
|
136
|
+
* @return RAC_SUCCESS or error code
|
|
137
|
+
*/
|
|
138
|
+
RAC_API rac_result_t rac_diffusion_generate_with_progress(
|
|
139
|
+
rac_handle_t handle, const rac_diffusion_options_t* options,
|
|
140
|
+
rac_diffusion_progress_callback_fn progress_callback, void* user_data,
|
|
141
|
+
rac_diffusion_result_t* out_result);
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* @brief Get service information
|
|
145
|
+
*
|
|
146
|
+
* @param handle Service handle
|
|
147
|
+
* @param out_info Output: Service information
|
|
148
|
+
* @return RAC_SUCCESS or error code
|
|
149
|
+
*/
|
|
150
|
+
RAC_API rac_result_t rac_diffusion_get_info(rac_handle_t handle, rac_diffusion_info_t* out_info);
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* @brief Get supported capabilities as bitmask
|
|
154
|
+
*
|
|
155
|
+
* @param handle Service handle
|
|
156
|
+
* @return Capability bitmask (RAC_DIFFUSION_CAP_* flags)
|
|
157
|
+
*/
|
|
158
|
+
RAC_API uint32_t rac_diffusion_get_capabilities(rac_handle_t handle);
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* @brief Cancel ongoing generation
|
|
162
|
+
*
|
|
163
|
+
* @param handle Service handle
|
|
164
|
+
* @return RAC_SUCCESS or error code
|
|
165
|
+
*/
|
|
166
|
+
RAC_API rac_result_t rac_diffusion_cancel(rac_handle_t handle);
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* @brief Cleanup and release model resources
|
|
170
|
+
*
|
|
171
|
+
* @param handle Service handle
|
|
172
|
+
* @return RAC_SUCCESS or error code
|
|
173
|
+
*/
|
|
174
|
+
RAC_API rac_result_t rac_diffusion_cleanup(rac_handle_t handle);
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* @brief Destroy a diffusion service instance
|
|
178
|
+
*
|
|
179
|
+
* @param handle Service handle to destroy
|
|
180
|
+
*/
|
|
181
|
+
RAC_API void rac_diffusion_destroy(rac_handle_t handle);
|
|
182
|
+
|
|
183
|
+
#ifdef __cplusplus
|
|
184
|
+
}
|
|
185
|
+
#endif
|
|
186
|
+
|
|
187
|
+
#endif /* RAC_DIFFUSION_SERVICE_H */
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file rac_diffusion_tokenizer.h
|
|
3
|
+
* @brief RunAnywhere Commons - Diffusion Tokenizer Utilities
|
|
4
|
+
*
|
|
5
|
+
* Utilities for managing diffusion model tokenizer files.
|
|
6
|
+
* Apple's compiled CoreML models don't include tokenizer files (vocab.json, merges.txt),
|
|
7
|
+
* so they must be downloaded from HuggingFace.
|
|
8
|
+
*
|
|
9
|
+
* This API provides:
|
|
10
|
+
* - URL resolution for predefined tokenizer sources
|
|
11
|
+
* - Automatic download of missing tokenizer files
|
|
12
|
+
* - Support for custom tokenizer URLs
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
#ifndef RAC_DIFFUSION_TOKENIZER_H
|
|
16
|
+
#define RAC_DIFFUSION_TOKENIZER_H
|
|
17
|
+
|
|
18
|
+
#include <RACommons/rac_types.h>
|
|
19
|
+
#include <RACommons/rac_diffusion_types.h>
|
|
20
|
+
|
|
21
|
+
#ifdef __cplusplus
|
|
22
|
+
extern "C" {
|
|
23
|
+
#endif
|
|
24
|
+
|
|
25
|
+
// =============================================================================
|
|
26
|
+
// TOKENIZER FILE NAMES
|
|
27
|
+
// =============================================================================
|
|
28
|
+
|
|
29
|
+
/** Vocabulary file name */
|
|
30
|
+
#define RAC_DIFFUSION_TOKENIZER_VOCAB_FILE "vocab.json"
|
|
31
|
+
|
|
32
|
+
/** Merge rules file name */
|
|
33
|
+
#define RAC_DIFFUSION_TOKENIZER_MERGES_FILE "merges.txt"
|
|
34
|
+
|
|
35
|
+
// =============================================================================
|
|
36
|
+
// URL RESOLUTION
|
|
37
|
+
// =============================================================================
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* @brief Get the base URL for a tokenizer source
|
|
41
|
+
*
|
|
42
|
+
* Returns the HuggingFace URL for the specified tokenizer source.
|
|
43
|
+
* For custom sources, returns the custom_base_url from config.
|
|
44
|
+
*
|
|
45
|
+
* @param source Tokenizer source preset
|
|
46
|
+
* @param custom_url Custom URL (only used when source == RAC_DIFFUSION_TOKENIZER_CUSTOM)
|
|
47
|
+
* @return Base URL string (static, do not free), or NULL if invalid
|
|
48
|
+
*
|
|
49
|
+
* @note URLs returned are HuggingFace raw file URLs (resolve/main/tokenizer)
|
|
50
|
+
*
|
|
51
|
+
* Example return values:
|
|
52
|
+
* - SD_1_5: "https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/tokenizer"
|
|
53
|
+
* - SD_2_X: "https://huggingface.co/stabilityai/stable-diffusion-2-1/resolve/main/tokenizer"
|
|
54
|
+
* - SDXL: "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/tokenizer"
|
|
55
|
+
* - CUSTOM: Returns custom_url parameter
|
|
56
|
+
*/
|
|
57
|
+
RAC_API const char* rac_diffusion_tokenizer_get_base_url(rac_diffusion_tokenizer_source_t source,
|
|
58
|
+
const char* custom_url);
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* @brief Get the full URL for a tokenizer file
|
|
62
|
+
*
|
|
63
|
+
* Constructs the full URL for downloading a specific tokenizer file.
|
|
64
|
+
*
|
|
65
|
+
* @param source Tokenizer source preset
|
|
66
|
+
* @param custom_url Custom URL (only used when source == RAC_DIFFUSION_TOKENIZER_CUSTOM)
|
|
67
|
+
* @param filename File name to append (e.g., "vocab.json" or "merges.txt")
|
|
68
|
+
* @param out_url Output buffer for the full URL
|
|
69
|
+
* @param out_url_size Size of output buffer
|
|
70
|
+
* @return RAC_SUCCESS or error code
|
|
71
|
+
*
|
|
72
|
+
* Example:
|
|
73
|
+
* @code
|
|
74
|
+
* char url[512];
|
|
75
|
+
* rac_diffusion_tokenizer_get_file_url(
|
|
76
|
+
* RAC_DIFFUSION_TOKENIZER_SD_1_5,
|
|
77
|
+
* NULL,
|
|
78
|
+
* "vocab.json",
|
|
79
|
+
* url,
|
|
80
|
+
* sizeof(url)
|
|
81
|
+
* );
|
|
82
|
+
* // url = "https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/tokenizer/vocab.json"
|
|
83
|
+
* @endcode
|
|
84
|
+
*/
|
|
85
|
+
RAC_API rac_result_t rac_diffusion_tokenizer_get_file_url(rac_diffusion_tokenizer_source_t source,
|
|
86
|
+
const char* custom_url,
|
|
87
|
+
const char* filename, char* out_url,
|
|
88
|
+
size_t out_url_size);
|
|
89
|
+
|
|
90
|
+
// =============================================================================
|
|
91
|
+
// FILE MANAGEMENT
|
|
92
|
+
// =============================================================================
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* @brief Check if tokenizer files exist in a directory
|
|
96
|
+
*
|
|
97
|
+
* @param model_dir Path to the model directory
|
|
98
|
+
* @param out_has_vocab Output: true if vocab.json exists
|
|
99
|
+
* @param out_has_merges Output: true if merges.txt exists
|
|
100
|
+
* @return RAC_SUCCESS or error code
|
|
101
|
+
*/
|
|
102
|
+
RAC_API rac_result_t rac_diffusion_tokenizer_check_files(const char* model_dir,
|
|
103
|
+
rac_bool_t* out_has_vocab,
|
|
104
|
+
rac_bool_t* out_has_merges);
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* @brief Ensure tokenizer files exist, downloading if necessary
|
|
108
|
+
*
|
|
109
|
+
* Checks for vocab.json and merges.txt in the model directory.
|
|
110
|
+
* If missing and auto_download is enabled, downloads from the configured source.
|
|
111
|
+
*
|
|
112
|
+
* @param model_dir Path to the model directory
|
|
113
|
+
* @param config Tokenizer configuration (source, custom URL, auto_download)
|
|
114
|
+
* @return RAC_SUCCESS if files exist or were downloaded successfully
|
|
115
|
+
* RAC_ERROR_FILE_NOT_FOUND if files missing and auto_download disabled
|
|
116
|
+
* RAC_ERROR_NETWORK if download failed
|
|
117
|
+
*
|
|
118
|
+
* Example:
|
|
119
|
+
* @code
|
|
120
|
+
* rac_diffusion_tokenizer_config_t config = {
|
|
121
|
+
* .source = RAC_DIFFUSION_TOKENIZER_SD_1_5,
|
|
122
|
+
* .custom_base_url = NULL,
|
|
123
|
+
* .auto_download = RAC_TRUE
|
|
124
|
+
* };
|
|
125
|
+
* rac_result_t result = rac_diffusion_tokenizer_ensure_files("/path/to/model", &config);
|
|
126
|
+
* @endcode
|
|
127
|
+
*/
|
|
128
|
+
RAC_API rac_result_t
|
|
129
|
+
rac_diffusion_tokenizer_ensure_files(const char* model_dir,
|
|
130
|
+
const rac_diffusion_tokenizer_config_t* config);
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* @brief Download a tokenizer file
|
|
134
|
+
*
|
|
135
|
+
* Downloads a specific tokenizer file from the configured source.
|
|
136
|
+
*
|
|
137
|
+
* @param source Tokenizer source preset
|
|
138
|
+
* @param custom_url Custom URL (only used when source == RAC_DIFFUSION_TOKENIZER_CUSTOM)
|
|
139
|
+
* @param filename File name to download (e.g., "vocab.json")
|
|
140
|
+
* @param output_path Full path where the file should be saved
|
|
141
|
+
* @return RAC_SUCCESS or error code
|
|
142
|
+
*/
|
|
143
|
+
RAC_API rac_result_t rac_diffusion_tokenizer_download_file(rac_diffusion_tokenizer_source_t source,
|
|
144
|
+
const char* custom_url,
|
|
145
|
+
const char* filename,
|
|
146
|
+
const char* output_path);
|
|
147
|
+
|
|
148
|
+
// =============================================================================
|
|
149
|
+
// DEFAULT TOKENIZER SOURCE FOR MODEL VARIANT
|
|
150
|
+
// =============================================================================
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* @brief Get the default tokenizer source for a model variant
|
|
154
|
+
*
|
|
155
|
+
* Returns the recommended tokenizer source for a given model variant.
|
|
156
|
+
*
|
|
157
|
+
* @param model_variant Model variant (SD 1.5, SD 2.1, SDXL, etc.)
|
|
158
|
+
* @return Default tokenizer source
|
|
159
|
+
*/
|
|
160
|
+
RAC_API rac_diffusion_tokenizer_source_t
|
|
161
|
+
rac_diffusion_tokenizer_default_for_variant(rac_diffusion_model_variant_t model_variant);
|
|
162
|
+
|
|
163
|
+
#ifdef __cplusplus
|
|
164
|
+
}
|
|
165
|
+
#endif
|
|
166
|
+
|
|
167
|
+
#endif /* RAC_DIFFUSION_TOKENIZER_H */
|