cactus-react-native 1.2.1 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (214) hide show
  1. package/README.md +555 -8
  2. package/android/CMakeLists.txt +4 -3
  3. package/android/src/main/java/com/margelo/nitro/cactus/HybridCactusFileSystem.kt +20 -1
  4. package/android/src/main/jniLibs/arm64-v8a/libcactus.a +0 -0
  5. package/android/src/main/jniLibs/arm64-v8a/libcactus_util.a +0 -0
  6. package/cpp/HybridCactus.cpp +112 -19
  7. package/cpp/HybridCactus.hpp +12 -3
  8. package/cpp/HybridCactusIndex.cpp +325 -0
  9. package/cpp/HybridCactusIndex.hpp +43 -0
  10. package/cpp/HybridCactusUtil.cpp +3 -3
  11. package/cpp/HybridCactusUtil.hpp +2 -1
  12. package/cpp/cactus_ffi.h +83 -2
  13. package/cpp/cactus_util.h +1 -1
  14. package/ios/HybridCactusFileSystem.swift +23 -2
  15. package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/cactus.h +2 -0
  16. package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/cactus_ffi.h +83 -2
  17. package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/cactus_telemetry.h +656 -0
  18. package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/{ffi_utils.h → cactus_utils.h} +104 -17
  19. package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/engine.h +117 -7
  20. package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/graph.h +91 -5
  21. package/ios/cactus.xcframework/ios-arm64/cactus.framework/Headers/kernel.h +15 -6
  22. package/ios/cactus.xcframework/ios-arm64/cactus.framework/Info.plist +0 -0
  23. package/ios/cactus.xcframework/ios-arm64/cactus.framework/cactus +0 -0
  24. package/ios/cactus.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus.h +2 -0
  25. package/ios/cactus.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_ffi.h +83 -2
  26. package/ios/cactus.xcframework/ios-arm64-simulator/cactus.framework/Headers/cactus_telemetry.h +656 -0
  27. package/ios/cactus.xcframework/ios-arm64-simulator/cactus.framework/Headers/{ffi_utils.h → cactus_utils.h} +104 -17
  28. package/ios/cactus.xcframework/ios-arm64-simulator/cactus.framework/Headers/engine.h +117 -7
  29. package/ios/cactus.xcframework/ios-arm64-simulator/cactus.framework/Headers/graph.h +91 -5
  30. package/ios/cactus.xcframework/ios-arm64-simulator/cactus.framework/Headers/kernel.h +15 -6
  31. package/ios/cactus.xcframework/ios-arm64-simulator/cactus.framework/Info.plist +0 -0
  32. package/ios/cactus.xcframework/ios-arm64-simulator/cactus.framework/_CodeSignature/CodeResources +1 -1
  33. package/ios/cactus.xcframework/ios-arm64-simulator/cactus.framework/cactus +0 -0
  34. package/ios/cactus_util.xcframework/Info.plist +4 -4
  35. package/ios/cactus_util.xcframework/ios-arm64/cactus_util.framework/Headers/cactus_util.h +1 -1
  36. package/ios/cactus_util.xcframework/ios-arm64/cactus_util.framework/Headers/database.h +27 -0
  37. package/ios/cactus_util.xcframework/ios-arm64/cactus_util.framework/Info.plist +0 -0
  38. package/ios/cactus_util.xcframework/ios-arm64/cactus_util.framework/cactus_util +0 -0
  39. package/ios/cactus_util.xcframework/ios-arm64-simulator/cactus_util.framework/Headers/cactus_util.h +1 -1
  40. package/ios/cactus_util.xcframework/ios-arm64-simulator/cactus_util.framework/Headers/database.h +27 -0
  41. package/ios/cactus_util.xcframework/ios-arm64-simulator/cactus_util.framework/Info.plist +0 -0
  42. package/ios/cactus_util.xcframework/ios-arm64-simulator/cactus_util.framework/_CodeSignature/CodeResources +3 -3
  43. package/ios/cactus_util.xcframework/ios-arm64-simulator/cactus_util.framework/cactus_util +0 -0
  44. package/lib/module/api/Database.js +12 -3
  45. package/lib/module/api/Database.js.map +1 -1
  46. package/lib/module/classes/CactusIndex.js +45 -0
  47. package/lib/module/classes/CactusIndex.js.map +1 -0
  48. package/lib/module/classes/CactusLM.js +35 -5
  49. package/lib/module/classes/CactusLM.js.map +1 -1
  50. package/lib/module/classes/CactusSTT.js +17 -5
  51. package/lib/module/classes/CactusSTT.js.map +1 -1
  52. package/lib/module/config/CactusConfig.js +2 -0
  53. package/lib/module/config/CactusConfig.js.map +1 -1
  54. package/lib/module/constants/packageVersion.js +1 -1
  55. package/lib/module/hooks/useCactusIndex.js +175 -0
  56. package/lib/module/hooks/useCactusIndex.js.map +1 -0
  57. package/lib/module/hooks/useCactusLM.js +54 -2
  58. package/lib/module/hooks/useCactusLM.js.map +1 -1
  59. package/lib/module/hooks/useCactusSTT.js +2 -2
  60. package/lib/module/hooks/useCactusSTT.js.map +1 -1
  61. package/lib/module/index.js +2 -0
  62. package/lib/module/index.js.map +1 -1
  63. package/lib/module/native/Cactus.js +24 -13
  64. package/lib/module/native/Cactus.js.map +1 -1
  65. package/lib/module/native/CactusFileSystem.js +3 -0
  66. package/lib/module/native/CactusFileSystem.js.map +1 -1
  67. package/lib/module/native/CactusIndex.js +32 -0
  68. package/lib/module/native/CactusIndex.js.map +1 -0
  69. package/lib/module/native/CactusUtil.js +16 -3
  70. package/lib/module/native/CactusUtil.js.map +1 -1
  71. package/lib/module/native/index.js +1 -0
  72. package/lib/module/native/index.js.map +1 -1
  73. package/lib/module/specs/CactusIndex.nitro.js +4 -0
  74. package/lib/module/specs/CactusIndex.nitro.js.map +1 -0
  75. package/lib/module/telemetry/Telemetry.js +3 -1
  76. package/lib/module/telemetry/Telemetry.js.map +1 -1
  77. package/lib/module/types/CactusIndex.js +2 -0
  78. package/lib/module/types/CactusIndex.js.map +1 -0
  79. package/lib/typescript/src/api/Database.d.ts +4 -1
  80. package/lib/typescript/src/api/Database.d.ts.map +1 -1
  81. package/lib/typescript/src/classes/CactusIndex.d.ts +15 -0
  82. package/lib/typescript/src/classes/CactusIndex.d.ts.map +1 -0
  83. package/lib/typescript/src/classes/CactusLM.d.ts +5 -2
  84. package/lib/typescript/src/classes/CactusLM.d.ts.map +1 -1
  85. package/lib/typescript/src/classes/CactusSTT.d.ts +2 -1
  86. package/lib/typescript/src/classes/CactusSTT.d.ts.map +1 -1
  87. package/lib/typescript/src/config/CactusConfig.d.ts +1 -0
  88. package/lib/typescript/src/config/CactusConfig.d.ts.map +1 -1
  89. package/lib/typescript/src/constants/packageVersion.d.ts +1 -1
  90. package/lib/typescript/src/hooks/useCactusIndex.d.ts +14 -0
  91. package/lib/typescript/src/hooks/useCactusIndex.d.ts.map +1 -0
  92. package/lib/typescript/src/hooks/useCactusLM.d.ts +4 -2
  93. package/lib/typescript/src/hooks/useCactusLM.d.ts.map +1 -1
  94. package/lib/typescript/src/hooks/useCactusSTT.d.ts +1 -1
  95. package/lib/typescript/src/hooks/useCactusSTT.d.ts.map +1 -1
  96. package/lib/typescript/src/index.d.ts +4 -1
  97. package/lib/typescript/src/index.d.ts.map +1 -1
  98. package/lib/typescript/src/native/Cactus.d.ts +4 -2
  99. package/lib/typescript/src/native/Cactus.d.ts.map +1 -1
  100. package/lib/typescript/src/native/CactusFileSystem.d.ts +1 -0
  101. package/lib/typescript/src/native/CactusFileSystem.d.ts.map +1 -1
  102. package/lib/typescript/src/native/CactusIndex.d.ts +12 -0
  103. package/lib/typescript/src/native/CactusIndex.d.ts.map +1 -0
  104. package/lib/typescript/src/native/CactusUtil.d.ts.map +1 -1
  105. package/lib/typescript/src/native/index.d.ts +1 -0
  106. package/lib/typescript/src/native/index.d.ts.map +1 -1
  107. package/lib/typescript/src/specs/Cactus.nitro.d.ts +4 -2
  108. package/lib/typescript/src/specs/Cactus.nitro.d.ts.map +1 -1
  109. package/lib/typescript/src/specs/CactusFileSystem.nitro.d.ts +1 -0
  110. package/lib/typescript/src/specs/CactusFileSystem.nitro.d.ts.map +1 -1
  111. package/lib/typescript/src/specs/CactusIndex.nitro.d.ts +24 -0
  112. package/lib/typescript/src/specs/CactusIndex.nitro.d.ts.map +1 -0
  113. package/lib/typescript/src/specs/CactusUtil.nitro.d.ts +1 -1
  114. package/lib/typescript/src/specs/CactusUtil.nitro.d.ts.map +1 -1
  115. package/lib/typescript/src/types/CactusIndex.d.ts +34 -0
  116. package/lib/typescript/src/types/CactusIndex.d.ts.map +1 -0
  117. package/lib/typescript/src/types/CactusLM.d.ts +17 -0
  118. package/lib/typescript/src/types/CactusLM.d.ts.map +1 -1
  119. package/lib/typescript/src/types/CactusSTT.d.ts +1 -1
  120. package/lib/typescript/src/types/CactusSTT.d.ts.map +1 -1
  121. package/nitro.json +3 -0
  122. package/nitrogen/generated/android/c++/JDeviceInfo.hpp +1 -1
  123. package/nitrogen/generated/android/c++/JFunc_void_double.hpp +1 -1
  124. package/nitrogen/generated/android/c++/JHybridCactusCryptoSpec.cpp +1 -1
  125. package/nitrogen/generated/android/c++/JHybridCactusCryptoSpec.hpp +1 -1
  126. package/nitrogen/generated/android/c++/JHybridCactusDeviceInfoSpec.cpp +1 -1
  127. package/nitrogen/generated/android/c++/JHybridCactusDeviceInfoSpec.hpp +1 -1
  128. package/nitrogen/generated/android/c++/JHybridCactusFileSystemSpec.cpp +17 -1
  129. package/nitrogen/generated/android/c++/JHybridCactusFileSystemSpec.hpp +2 -1
  130. package/nitrogen/generated/android/c++/JHybridCactusImageSpec.cpp +1 -1
  131. package/nitrogen/generated/android/c++/JHybridCactusImageSpec.hpp +1 -1
  132. package/nitrogen/generated/android/cactus+autolinking.cmake +2 -1
  133. package/nitrogen/generated/android/cactus+autolinking.gradle +1 -1
  134. package/nitrogen/generated/android/cactusOnLoad.cpp +11 -1
  135. package/nitrogen/generated/android/cactusOnLoad.hpp +1 -1
  136. package/nitrogen/generated/android/kotlin/com/margelo/nitro/cactus/DeviceInfo.kt +1 -1
  137. package/nitrogen/generated/android/kotlin/com/margelo/nitro/cactus/Func_void_double.kt +1 -1
  138. package/nitrogen/generated/android/kotlin/com/margelo/nitro/cactus/HybridCactusCryptoSpec.kt +1 -1
  139. package/nitrogen/generated/android/kotlin/com/margelo/nitro/cactus/HybridCactusDeviceInfoSpec.kt +1 -1
  140. package/nitrogen/generated/android/kotlin/com/margelo/nitro/cactus/HybridCactusFileSystemSpec.kt +5 -1
  141. package/nitrogen/generated/android/kotlin/com/margelo/nitro/cactus/HybridCactusImageSpec.kt +1 -1
  142. package/nitrogen/generated/android/kotlin/com/margelo/nitro/cactus/cactusOnLoad.kt +1 -1
  143. package/nitrogen/generated/ios/Cactus+autolinking.rb +1 -1
  144. package/nitrogen/generated/ios/Cactus-Swift-Cxx-Bridge.cpp +1 -1
  145. package/nitrogen/generated/ios/Cactus-Swift-Cxx-Bridge.hpp +1 -1
  146. package/nitrogen/generated/ios/Cactus-Swift-Cxx-Umbrella.hpp +1 -1
  147. package/nitrogen/generated/ios/CactusAutolinking.mm +11 -1
  148. package/nitrogen/generated/ios/CactusAutolinking.swift +1 -1
  149. package/nitrogen/generated/ios/c++/HybridCactusCryptoSpecSwift.cpp +1 -1
  150. package/nitrogen/generated/ios/c++/HybridCactusCryptoSpecSwift.hpp +1 -1
  151. package/nitrogen/generated/ios/c++/HybridCactusDeviceInfoSpecSwift.cpp +1 -1
  152. package/nitrogen/generated/ios/c++/HybridCactusDeviceInfoSpecSwift.hpp +1 -1
  153. package/nitrogen/generated/ios/c++/HybridCactusFileSystemSpecSwift.cpp +1 -1
  154. package/nitrogen/generated/ios/c++/HybridCactusFileSystemSpecSwift.hpp +9 -1
  155. package/nitrogen/generated/ios/c++/HybridCactusImageSpecSwift.cpp +1 -1
  156. package/nitrogen/generated/ios/c++/HybridCactusImageSpecSwift.hpp +1 -1
  157. package/nitrogen/generated/ios/swift/DeviceInfo.swift +1 -1
  158. package/nitrogen/generated/ios/swift/Func_void.swift +1 -1
  159. package/nitrogen/generated/ios/swift/Func_void_DeviceInfo.swift +1 -1
  160. package/nitrogen/generated/ios/swift/Func_void_bool.swift +1 -1
  161. package/nitrogen/generated/ios/swift/Func_void_double.swift +1 -1
  162. package/nitrogen/generated/ios/swift/Func_void_std__exception_ptr.swift +1 -1
  163. package/nitrogen/generated/ios/swift/Func_void_std__optional_std__string_.swift +1 -1
  164. package/nitrogen/generated/ios/swift/Func_void_std__string.swift +1 -1
  165. package/nitrogen/generated/ios/swift/HybridCactusCryptoSpec.swift +1 -1
  166. package/nitrogen/generated/ios/swift/HybridCactusCryptoSpec_cxx.swift +1 -1
  167. package/nitrogen/generated/ios/swift/HybridCactusDeviceInfoSpec.swift +1 -1
  168. package/nitrogen/generated/ios/swift/HybridCactusDeviceInfoSpec_cxx.swift +1 -1
  169. package/nitrogen/generated/ios/swift/HybridCactusFileSystemSpec.swift +2 -1
  170. package/nitrogen/generated/ios/swift/HybridCactusFileSystemSpec_cxx.swift +20 -1
  171. package/nitrogen/generated/ios/swift/HybridCactusImageSpec.swift +1 -1
  172. package/nitrogen/generated/ios/swift/HybridCactusImageSpec_cxx.swift +1 -1
  173. package/nitrogen/generated/shared/c++/CactusIndexGetResult.hpp +84 -0
  174. package/nitrogen/generated/shared/c++/CactusIndexQueryResult.hpp +79 -0
  175. package/nitrogen/generated/shared/c++/DeviceInfo.hpp +1 -1
  176. package/nitrogen/generated/shared/c++/HybridCactusCryptoSpec.cpp +1 -1
  177. package/nitrogen/generated/shared/c++/HybridCactusCryptoSpec.hpp +1 -1
  178. package/nitrogen/generated/shared/c++/HybridCactusDeviceInfoSpec.cpp +1 -1
  179. package/nitrogen/generated/shared/c++/HybridCactusDeviceInfoSpec.hpp +1 -1
  180. package/nitrogen/generated/shared/c++/HybridCactusFileSystemSpec.cpp +2 -1
  181. package/nitrogen/generated/shared/c++/HybridCactusFileSystemSpec.hpp +2 -1
  182. package/nitrogen/generated/shared/c++/HybridCactusImageSpec.cpp +1 -1
  183. package/nitrogen/generated/shared/c++/HybridCactusImageSpec.hpp +1 -1
  184. package/nitrogen/generated/shared/c++/HybridCactusIndexSpec.cpp +27 -0
  185. package/nitrogen/generated/shared/c++/HybridCactusIndexSpec.hpp +76 -0
  186. package/nitrogen/generated/shared/c++/HybridCactusSpec.cpp +3 -1
  187. package/nitrogen/generated/shared/c++/HybridCactusSpec.hpp +6 -3
  188. package/nitrogen/generated/shared/c++/HybridCactusUtilSpec.cpp +1 -1
  189. package/nitrogen/generated/shared/c++/HybridCactusUtilSpec.hpp +2 -2
  190. package/package.json +2 -2
  191. package/src/api/Database.ts +14 -2
  192. package/src/classes/CactusIndex.ts +58 -0
  193. package/src/classes/CactusLM.ts +41 -5
  194. package/src/classes/CactusSTT.ts +19 -6
  195. package/src/config/CactusConfig.ts +3 -0
  196. package/src/constants/packageVersion.ts +1 -1
  197. package/src/hooks/useCactusIndex.ts +195 -0
  198. package/src/hooks/useCactusLM.ts +63 -3
  199. package/src/hooks/useCactusSTT.ts +2 -2
  200. package/src/index.tsx +16 -0
  201. package/src/native/Cactus.ts +40 -13
  202. package/src/native/CactusFileSystem.ts +4 -0
  203. package/src/native/CactusIndex.ts +54 -0
  204. package/src/native/CactusUtil.ts +19 -3
  205. package/src/native/index.ts +1 -0
  206. package/src/specs/Cactus.nitro.ts +13 -2
  207. package/src/specs/CactusFileSystem.nitro.ts +2 -0
  208. package/src/specs/CactusIndex.nitro.ts +31 -0
  209. package/src/specs/CactusUtil.nitro.ts +1 -1
  210. package/src/telemetry/Telemetry.ts +1 -1
  211. package/src/types/CactusIndex.ts +40 -0
  212. package/src/types/CactusLM.ts +21 -0
  213. package/src/types/CactusSTT.ts +1 -1
  214. package/android/src/main/jniLibs/arm64-v8a/libcactus_util.so +0 -0
@@ -1,5 +1,5 @@
1
- #ifndef CACTUS_FFI_UTILS_H
2
- #define CACTUS_FFI_UTILS_H
1
+ #ifndef CACTUS_UTILS_H
2
+ #define CACTUS_UTILS_H
3
3
 
4
4
  #include "../engine/engine.h"
5
5
  #include <string>
@@ -12,10 +12,51 @@
12
12
  #include <iostream>
13
13
  #include <filesystem>
14
14
  #include <cctype>
15
+ #include <memory>
16
+ #include <atomic>
17
+ #include <mutex>
18
+ #include <random>
19
+
20
+ #ifdef __APPLE__
21
+ #include <uuid/uuid.h>
22
+ #endif
23
+
24
+ struct CactusModelHandle {
25
+ std::unique_ptr<cactus::engine::Model> model;
26
+ std::atomic<bool> should_stop;
27
+ std::vector<uint32_t> processed_tokens;
28
+ std::mutex model_mutex;
29
+ std::string model_name;
30
+
31
+ CactusModelHandle() : should_stop(false) {}
32
+ };
33
+
34
+ extern std::string last_error_message;
35
+
36
+ bool matches_stop_sequence(const std::vector<uint32_t>& generated_tokens,
37
+ const std::vector<std::vector<uint32_t>>& stop_sequences);
15
38
 
16
39
  namespace cactus {
17
40
  namespace ffi {
18
41
 
42
+ #ifndef CACTUS_VERSION
43
+ #define CACTUS_VERSION "unknown"
44
+ #endif
45
+
46
+ inline const char* getVersion() {
47
+ return CACTUS_VERSION;
48
+ }
49
+
50
+ inline std::string generateUUID() {
51
+ #ifdef __APPLE__
52
+ uuid_t uuid;
53
+ uuid_generate_random(uuid);
54
+ char uuid_str[37];
55
+ uuid_unparse_lower(uuid, uuid_str);
56
+ return std::string(uuid_str);
57
+ #endif
58
+ }
59
+
19
60
  struct ToolFunction {
20
61
  std::string name;
21
62
  std::string description;
@@ -175,49 +216,58 @@ inline std::vector<ToolFunction> parse_tools_json(const std::string& json) {
175
216
  return tools;
176
217
  }
177
218
 
178
- inline void parse_options_json(const std::string& json,
179
- float& temperature, float& top_p,
219
+ inline void parse_options_json(const std::string& json,
220
+ float& temperature, float& top_p,
180
221
  size_t& top_k, size_t& max_tokens,
181
- std::vector<std::string>& stop_sequences) {
222
+ std::vector<std::string>& stop_sequences,
223
+ bool& force_tools) {
182
224
  temperature = 0.0f;
183
- top_p = 0.0f;
184
- top_k = 0;
185
- max_tokens = 100;
225
+ top_p = 0.0f;
226
+ top_k = 0;
227
+ max_tokens = 100;
228
+ force_tools = false;
186
229
  stop_sequences.clear();
187
-
230
+
188
231
  if (json.empty()) return;
189
-
232
+
190
233
  size_t pos = json.find("\"temperature\"");
191
234
  if (pos != std::string::npos) {
192
235
  pos = json.find(':', pos) + 1;
193
236
  temperature = std::stof(json.substr(pos));
194
237
  }
195
-
238
+
196
239
  pos = json.find("\"top_p\"");
197
240
  if (pos != std::string::npos) {
198
241
  pos = json.find(':', pos) + 1;
199
242
  top_p = std::stof(json.substr(pos));
200
243
  }
201
-
244
+
202
245
  pos = json.find("\"top_k\"");
203
246
  if (pos != std::string::npos) {
204
247
  pos = json.find(':', pos) + 1;
205
248
  top_k = std::stoul(json.substr(pos));
206
249
  }
207
-
250
+
208
251
  pos = json.find("\"max_tokens\"");
209
252
  if (pos != std::string::npos) {
210
253
  pos = json.find(':', pos) + 1;
211
254
  max_tokens = std::stoul(json.substr(pos));
212
255
  }
213
-
256
+
257
+ pos = json.find("\"force_tools\"");
258
+ if (pos != std::string::npos) {
259
+ pos = json.find(':', pos) + 1;
260
+ while (pos < json.length() && std::isspace(json[pos])) pos++;
261
+ force_tools = (json.substr(pos, 4) == "true");
262
+ }
263
+
214
264
  pos = json.find("\"stop_sequences\"");
215
265
  if (pos != std::string::npos) {
216
266
  pos = json.find('[', pos);
217
267
  if (pos != std::string::npos) {
218
268
  size_t end_pos = json.find(']', pos);
219
269
  size_t seq_pos = json.find('"', pos);
220
-
270
+
221
271
  while (seq_pos != std::string::npos && seq_pos < end_pos) {
222
272
  size_t seq_start = seq_pos + 1;
223
273
  size_t seq_end = json.find('"', seq_start);
@@ -234,7 +284,7 @@ inline std::string format_tools_for_prompt(const std::vector<ToolFunction>& tool
234
284
  if (tools.empty()) return "";
235
285
  std::string formatted_tools_json;
236
286
  for (size_t i = 0; i < tools.size(); i++) {
237
- if (i > 0) formatted_tools_json += ",\n";
287
+ if (i > 0) formatted_tools_json += "\n";
238
288
  formatted_tools_json += "{\"type\":\"function\",\"function\":{\"name\":\""
239
289
  + tools[i].name
240
290
  + "\",\"description\":\""
@@ -398,4 +448,41 @@ inline std::string construct_response_json(const std::string& regular_response,
398
448
  } // namespace ffi
399
449
  } // namespace cactus
400
450
 
401
- #endif // CACTUS_FFI_UTILS_H
451
+ #ifdef __cplusplus
452
+ extern "C" {
453
+ #endif
454
+
455
+ const char* cactus_get_last_error();
456
+
457
+ __attribute__((weak))
458
+ const char* register_app(const char* encrypted_data);
459
+
460
+ __attribute__((weak))
461
+ const char* get_device_id(const char* current_token);
462
+
463
+ #ifdef __cplusplus
464
+ }
465
+ #endif
466
+
467
+ #ifdef __cplusplus
468
+ extern "C" {
469
+
470
+ __attribute__((weak))
471
+ inline const char* register_app(const char* encrypted_data) {
472
+ (void)encrypted_data;
473
+ static thread_local std::string uuid_storage;
474
+ uuid_storage = cactus::ffi::generateUUID();
475
+ return uuid_storage.c_str();
476
+ }
477
+
478
+ __attribute__((weak))
479
+ inline const char* get_device_id(const char* current_token) {
480
+ (void)current_token;
481
+ static thread_local std::string uuid_storage;
482
+ uuid_storage = cactus::ffi::generateUUID();
483
+ return uuid_storage.c_str();
484
+ }
485
+ }
486
+ #endif
487
+
488
+ #endif // CACTUS_UTILS_H
@@ -3,6 +3,7 @@
3
3
  #include <vector>
4
4
  #include <string>
5
5
  #include <unordered_map>
6
+ #include <unordered_set>
6
7
  #include <memory>
7
8
  #include <cstdint>
8
9
 
@@ -32,6 +33,9 @@ extern "C" {
32
33
  class CactusGraph;
33
34
 
34
35
  namespace cactus {
36
+ namespace npu {
37
+ class NPUPrefill;
38
+ }
35
39
  namespace engine {
36
40
 
37
41
  class Siglip2Preprocessor;
@@ -349,6 +353,13 @@ struct KVCache {
349
353
  void update_from_graph(CactusGraph* gb, const std::vector<size_t>& k_nodes,
350
354
  const std::vector<size_t>& v_nodes, size_t seq_len,
351
355
  size_t num_layers, size_t kv_heads, size_t head_dim);
356
+
357
+ // Update KV cache from NPU prefill outputs
358
+ // NPU outputs are in shape [num_tokens, num_kv_heads, head_dim]
359
+ // This handles transposition to cache format and sliding window
360
+ void update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data,
361
+ size_t num_tokens, size_t kv_heads, size_t head_dim);
362
+
352
363
  bool is_empty() const { return current_seq_len == 0; }
353
364
  void* get_key_ptr(size_t layer);
354
365
  void* get_value_ptr(size_t layer);
@@ -365,6 +376,85 @@ struct KVCache {
365
376
  CircularView get_value_view(size_t layer);
366
377
  };
367
378
 
379
+ class ToolCallConstrainer {
380
+ public:
381
+ enum class State {
382
+ START, // -> expect {
383
+ EXPECT_FC_KEY, // -> expect "function_call"
384
+ EXPECT_FC_COLON, // -> expect :
385
+ EXPECT_FC_OPEN_BRACE, // -> expect {
386
+ EXPECT_NAME_KEY, // -> expect "name"
387
+ EXPECT_NAME_COLON, // -> expect :
388
+ EXPECT_NAME_VALUE, // -> expect "<function_name>"
389
+ EXPECT_COMMA, // -> expect ,
390
+ EXPECT_ARGS_KEY, // -> expect "arguments"
391
+ EXPECT_ARGS_COLON, // -> expect :
392
+ IN_ARGUMENTS, // -> free JSON, track brace depth
393
+ EXPECT_INNER_CLOSE, // -> expect } to close inner object
394
+ EXPECT_OUTER_CLOSE, // -> expect } to close outer object
395
+ DONE, // complete
396
+
397
+ LFM_START, // -> expect <|tool_call_start|>
398
+ LFM_EXPECT_BRACKET, // -> expect [
399
+ LFM_IN_FUNC_NAME, // -> expect function name
400
+ LFM_EXPECT_PAREN, // -> expect (
401
+ LFM_IN_ARGUMENTS, // -> arguments until )
402
+ LFM_EXPECT_BRACKET_CLOSE, // -> expect ]
403
+ LFM_EXPECT_END // -> expect <|tool_call_end|>
404
+ };
405
+
406
+ void init(Config::ModelType model_type,
407
+ const std::vector<std::string>& function_names,
408
+ Tokenizer* tokenizer);
409
+
410
+ const std::unordered_map<uint32_t, float>& get_bias() const { return current_bias_; }
411
+
412
+ void update(uint32_t token_id, const std::string& decoded_text);
413
+
414
+ void reset();
415
+
416
+ bool is_active() const { return active_; }
417
+
418
+ private:
419
+ bool active_ = false;
420
+ State state_ = State::START;
421
+ Config::ModelType model_type_ = Config::ModelType::QWEN;
422
+ Tokenizer* tokenizer_ = nullptr;
423
+
424
+ std::vector<std::string> function_names_;
425
+ std::string generated_text_;
426
+ int brace_depth_ = 0; // Track nested braces in arguments
427
+
428
+ // Pre-tokenized token sets for each grammar element
429
+ std::unordered_set<uint32_t> open_brace_tokens_; // {
430
+ std::unordered_set<uint32_t> close_brace_tokens_; // }
431
+ std::unordered_set<uint32_t> colon_tokens_; // :
432
+ std::unordered_set<uint32_t> comma_tokens_; // ,
433
+ std::unordered_set<uint32_t> fc_key_tokens_; // "function_call"
434
+ std::unordered_set<uint32_t> name_key_tokens_; // "name"
435
+ std::unordered_set<uint32_t> args_key_tokens_; // "arguments"
436
+ std::unordered_set<uint32_t> quote_tokens_; // "
437
+ std::unordered_set<uint32_t> backtick_tokens_; // ` (to block markdown code fences)
438
+ std::unordered_set<uint32_t> response_starter_tokens_; // Common response starters to block (I, I'm, Sorry, etc.)
439
+ std::unordered_set<uint32_t> all_func_name_tokens_; // All function name tokens combined
440
+ std::unordered_map<std::string, std::vector<uint32_t>> func_name_sequences_; // Full token sequence per function
441
+
442
+ // LFM2-specific tokens
443
+ std::unordered_set<uint32_t> tool_start_tokens_;
444
+ std::unordered_set<uint32_t> tool_end_tokens_;
445
+ std::unordered_set<uint32_t> bracket_open_tokens_; // [
446
+ std::unordered_set<uint32_t> bracket_close_tokens_; // ]
447
+ std::unordered_set<uint32_t> paren_open_tokens_; // (
448
+ std::unordered_set<uint32_t> paren_close_tokens_; // )
449
+ std::unordered_set<uint32_t> equals_tokens_; // =
450
+
451
+ std::unordered_map<uint32_t, float> current_bias_;
452
+
453
+ void compute_bias();
454
+ void tokenize_grammar_elements();
455
+ void add_tokens_for_string(const std::string& str, std::unordered_set<uint32_t>& token_set);
456
+ };
457
+
368
458
  class Model {
369
459
  public:
370
460
  struct DebugNode {
@@ -386,26 +476,40 @@ public:
386
476
  virtual bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size,
387
477
  const std::string& system_prompt = "", bool do_warmup = true);
388
478
 
389
- virtual uint32_t generate(const std::vector<uint32_t>& tokens, float temperature = -1.0f, float top_p = -1.0f,
390
- size_t top_k = 0, const std::string& profile_file = "", bool prefill_only = false);
479
+ virtual uint32_t decode(const std::vector<uint32_t>& tokens, float temperature = -1.0f, float top_p = -1.0f,
480
+ size_t top_k = 0, const std::string& profile_file = "");
481
+
482
+ virtual void prefill(const std::vector<uint32_t>& tokens, size_t chunk_size = 256, const std::string& profile_file = "");
391
483
 
392
- virtual uint32_t generate_with_images(const std::vector<uint32_t>& tokens, const std::vector<std::string>& image_paths,
484
+ virtual uint32_t decode_with_images(const std::vector<uint32_t>& tokens, const std::vector<std::string>& image_paths,
393
485
  float temperature = -1.0f, float top_p = -1.0f,
394
486
  size_t top_k = 0, const std::string& profile_file = "");
395
-
396
- virtual uint32_t generate_with_audio(const std::vector<uint32_t>& tokens, const std::vector<float>& mel_bins, float temperature = 0.0f, float top_p = 0.0f,
487
+
488
+ virtual uint32_t decode_with_audio(const std::vector<uint32_t>& tokens, const std::vector<float>& mel_bins, float temperature = 0.0f, float top_p = 0.0f,
397
489
  size_t top_k = 0, const std::string& profile_file = "");
398
490
 
399
- std::vector<float> get_embeddings(const std::vector<uint32_t>& tokens, bool pooled = true, const std::string& profile_file = "");
491
+ std::vector<float> get_embeddings(const std::vector<uint32_t>& tokens, bool pooled = true, bool normalize = false, const std::string& profile_file = "");
400
492
 
401
493
  virtual std::vector<float> get_image_embeddings(const std::string& image_path);
402
494
 
403
495
  virtual std::vector<float> get_audio_embeddings(const std::vector<float>& mel_bins);
404
496
 
405
497
  virtual void reset_cache() { kv_cache_.reset(); }
406
-
498
+
499
+ double score_tokens_window_logprob(const std::vector<uint32_t>& tokens, size_t start, size_t end, size_t context, size_t* tokens_scored);
500
+
501
+
502
+
407
503
  void set_cache_window(size_t window_size, size_t sink_size = 4) { kv_cache_.set_window_size(window_size, sink_size); }
408
504
 
505
+ bool load_npu_prefill(const std::string& model_path);
506
+ bool has_npu_prefill() const;
507
+ size_t get_prefill_chunk_size() const;
508
+
509
+ void set_tool_constraints(const std::vector<std::string>& function_names);
510
+ void clear_tool_constraints();
511
+ void update_tool_constraints(uint32_t token_id);
512
+
409
513
  void* graph_handle_;
410
514
 
411
515
  protected:
@@ -449,6 +553,12 @@ protected:
449
553
  bool init_internal(CactusGraph* gb, const std::string& model_folder, size_t context_size,
450
554
  const std::string& system_prompt, bool do_warmup);
451
555
  bool owns_graph_;
556
+
557
+ std::unique_ptr<npu::NPUPrefill> npu_prefill_;
558
+ void prefill_npu(const std::vector<uint32_t>& tokens);
559
+ virtual std::vector<__fp16> get_token_embeddings(const std::vector<uint32_t>& tokens);
560
+
561
+ ToolCallConstrainer tool_constrainer_;
452
562
  };
453
563
 
454
564
  std::unique_ptr<Model> create_model(const std::string& model_folder);
@@ -8,6 +8,88 @@
8
8
  #include <cstring>
9
9
  #include <stdexcept>
10
10
  #include <string>
11
+ #include <mutex>
12
+ #include <sstream>
13
+ #include <iostream>
14
+
15
+ namespace cactus {
16
+
17
+ enum class LogLevel {
18
+ DEBUG = 0,
19
+ INFO = 1,
20
+ WARN = 2,
21
+ ERROR = 3,
22
+ NONE = 4
23
+ };
24
+
25
+ class Logger {
26
+ public:
27
+ static Logger& instance() {
28
+ static Logger logger;
29
+ return logger;
30
+ }
31
+
32
+ void set_level(LogLevel level) { min_level_ = level; }
33
+ LogLevel get_level() const { return min_level_; }
34
+
35
+ void set_callback(std::function<void(LogLevel, const std::string&, const std::string&)> cb) {
36
+ std::lock_guard<std::mutex> lock(mutex_);
37
+ callback_ = cb;
38
+ }
39
+
40
+ void log(LogLevel level, const std::string& component, const std::string& message) {
41
+ if (level < min_level_) return;
42
+
43
+ std::lock_guard<std::mutex> lock(mutex_);
44
+
45
+ if (callback_) {
46
+ callback_(level, component, message);
47
+ } else {
48
+ std::cerr << "[" << level_string(level) << "] [" << component << "] " << message << std::endl;
49
+ }
50
+
51
+ if (level == LogLevel::ERROR) {
52
+ last_error_ = "[" + component + "] " + message;
53
+ }
54
+ }
55
+
56
+ const std::string& last_error() const { return last_error_; }
57
+ void clear_error() { last_error_.clear(); }
58
+
59
+ private:
60
+ Logger() : min_level_(LogLevel::WARN) {}
61
+
62
+ static const char* level_string(LogLevel level) {
63
+ switch (level) {
64
+ case LogLevel::DEBUG: return "DEBUG";
65
+ case LogLevel::INFO: return "INFO";
66
+ case LogLevel::WARN: return "WARN";
67
+ case LogLevel::ERROR: return "ERROR";
68
+ default: return "?";
69
+ }
70
+ }
71
+
72
+ LogLevel min_level_;
73
+ std::mutex mutex_;
74
+ std::string last_error_;
75
+ std::function<void(LogLevel, const std::string&, const std::string&)> callback_;
76
+ };
77
+
78
+ } // namespace cactus
79
+
80
+ #define CACTUS_LOG(level, component, msg) \
81
+ do { \
82
+ if (static_cast<int>(level) >= static_cast<int>(cactus::Logger::instance().get_level())) { \
83
+ std::ostringstream _cactus_log_ss; \
84
+ _cactus_log_ss << msg; \
85
+ cactus::Logger::instance().log(level, component, _cactus_log_ss.str()); \
86
+ } \
87
+ } while(0)
88
+
89
+ #define CACTUS_LOG_DEBUG(component, msg) CACTUS_LOG(cactus::LogLevel::DEBUG, component, msg)
90
+ #define CACTUS_LOG_INFO(component, msg) CACTUS_LOG(cactus::LogLevel::INFO, component, msg)
91
+ #define CACTUS_LOG_WARN(component, msg) CACTUS_LOG(cactus::LogLevel::WARN, component, msg)
92
+ #define CACTUS_LOG_ERROR(component, msg) CACTUS_LOG(cactus::LogLevel::ERROR, component, msg)
11
93
 
12
94
  namespace GraphFile {
13
95
  class MappedFile;
@@ -144,7 +226,7 @@ struct OpParams {
144
226
  size_t slice_start = 0;
145
227
  size_t slice_length = 0;
146
228
  size_t window_size = 0;
147
- bool is_causal = true; // Default to causal for backward compatibility
229
+ bool is_causal = true;
148
230
  std::vector<size_t> new_shape;
149
231
  std::vector<size_t> permutation;
150
232
  Precision output_precision = Precision::INT8;
@@ -158,10 +240,13 @@ struct OpParams {
158
240
  size_t top_k = 0;
159
241
  size_t random_seed = 0;
160
242
 
161
- size_t index_value = 0; // For INDEX operation
162
- size_t num_classes = 0; // For scatter operations
243
+ size_t index_value = 0;
244
+ size_t num_classes = 0;
163
245
  size_t dst_height = 0;
164
- size_t dst_width = 0;
246
+ size_t dst_width = 0;
247
+
248
+ std::vector<float> bias_values;
249
+ std::vector<uint32_t> bias_indices;
165
250
  };
166
251
 
167
252
  struct GraphNode {
@@ -294,7 +379,8 @@ public:
294
379
  size_t conv1d_causal(size_t input, size_t weight, size_t kernel_size, size_t dilation = 1);
295
380
  size_t conv1d_k3(size_t input, size_t weight, size_t stride);
296
381
 
297
- size_t sample(size_t logits, float temperature = 0.6f, float top_p = 0.95f, size_t top_k = 20);
382
+ size_t sample(size_t logits, float temperature = 0.6f, float top_p = 0.95f, size_t top_k = 20,
383
+ const std::unordered_map<uint32_t, float>& logit_bias = {});
298
384
 
299
385
  size_t concat(size_t input1, size_t input2, int axis = 0);
300
386
  size_t scatter_topk(size_t indices, size_t values, size_t num_classes);
@@ -102,18 +102,23 @@ void cactus_matmul_f32(const float* a, const float* b_transposed, float* c,
102
102
  size_t M, size_t K, size_t N);
103
103
 
104
104
 
105
- void cactus_transpose_2d_int8(const int8_t* source, int8_t* destination,
105
+ void cactus_transpose_2d_int8(const int8_t* source, int8_t* destination,
106
106
  size_t num_rows, size_t num_cols, size_t start_row, size_t end_row);
107
- void cactus_transpose_2d_f32(const float* source, float* destination,
107
+ void cactus_transpose_2d_f16(const __fp16* source, __fp16* destination,
108
+ size_t num_rows, size_t num_cols, size_t start_row, size_t end_row);
109
+ void cactus_transpose_2d_f32(const float* source, float* destination,
108
110
  size_t num_rows, size_t num_cols, size_t start_row, size_t end_row);
109
111
 
110
- void cactus_transpose_int8(const int8_t* source, int8_t* destination, const size_t* shape,
112
+ void cactus_transpose_int8(const int8_t* source, int8_t* destination, const size_t* shape,
111
113
  const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx);
112
- void cactus_transpose_f32(const float* source, float* destination, const size_t* shape,
114
+ void cactus_transpose_f16(const __fp16* source, __fp16* destination, const size_t* shape,
115
+ const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx);
116
+ void cactus_transpose_f32(const float* source, float* destination, const size_t* shape,
113
117
  const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx);
114
118
 
115
119
  int64_t cactus_sum_all_int8(const int8_t* data, size_t num_elements);
116
120
  void cactus_sum_axis_int8(const int8_t* input, int8_t* output, size_t outer_size, size_t axis_size, size_t inner_size);
121
+ double cactus_sum_all_f16(const __fp16* data, size_t num_elements);
117
122
  double cactus_sum_all_f32(const float* data, size_t num_elements);
118
123
  void cactus_sum_axis_f32(const float* input, float* output, size_t outer_size, size_t axis_size, size_t inner_size);
119
124
 
@@ -278,9 +283,13 @@ void cactus_bilinear_interpolation_fp32(const float* input, float* output, size_
278
283
  size_t dst_height, size_t dst_width);
279
284
 
280
285
  void cactus_sample_f32(const float* logits, uint32_t* output, size_t vocab_size,
281
- float temperature, float top_p, size_t top_k, size_t random_seed);
286
+ float temperature, float top_p, size_t top_k, size_t random_seed,
287
+ const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr,
288
+ size_t bias_count = 0);
282
289
  void cactus_sample_f16(const __fp16* logits, uint32_t* output, size_t vocab_size,
283
- float temperature, float top_p, size_t top_k, size_t random_seed);
290
+ float temperature, float top_p, size_t top_k, size_t random_seed,
291
+ const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr,
292
+ size_t bias_count = 0);
284
293
 
285
294
 
286
295
  void cactus_concat_f32(const float* input1, const float* input2, float* output,
@@ -6,7 +6,7 @@
6
6
  <dict>
7
7
  <key>Info.plist</key>
8
8
  <data>
9
- oZrZEx0iJfsyoH1nUXO8hrSVJHs=
9
+ yMSW0g+AKq/xXqUOrMbK43roF5I=
10
10
  </data>
11
11
  </dict>
12
12
  <key>files2</key>
@@ -6,7 +6,7 @@
6
6
  <array>
7
7
  <dict>
8
8
  <key>LibraryIdentifier</key>
9
- <string>ios-arm64-simulator</string>
9
+ <string>ios-arm64</string>
10
10
  <key>LibraryPath</key>
11
11
  <string>cactus_util.framework</string>
12
12
  <key>SupportedArchitectures</key>
@@ -15,12 +15,10 @@
15
15
  </array>
16
16
  <key>SupportedPlatform</key>
17
17
  <string>ios</string>
18
- <key>SupportedPlatformVariant</key>
19
- <string>simulator</string>
20
18
  </dict>
21
19
  <dict>
22
20
  <key>LibraryIdentifier</key>
23
- <string>ios-arm64</string>
21
+ <string>ios-arm64-simulator</string>
24
22
  <key>LibraryPath</key>
25
23
  <string>cactus_util.framework</string>
26
24
  <key>SupportedArchitectures</key>
@@ -29,6 +27,8 @@
29
27
  </array>
30
28
  <key>SupportedPlatform</key>
31
29
  <string>ios</string>
30
+ <key>SupportedPlatformVariant</key>
31
+ <string>simulator</string>
32
32
  </dict>
33
33
  </array>
34
34
  <key>CFBundlePackageType</key>
@@ -7,7 +7,7 @@ extern "C" {
7
7
 
8
8
  const char* register_app(const char* encrypted_data);
9
9
 
10
- const char* get_device_id();
10
+ const char* get_device_id(const char* current_token);
11
11
 
12
12
  // Helper function to free memory allocated by register_app
13
13
  void free_string(const char* str);
@@ -0,0 +1,27 @@
1
+ #ifndef DATABASE_H
2
+ #define DATABASE_H
3
+
4
+ #include <string>
5
+ struct sqlite3;
6
+
7
+ class Database {
8
+ private:
9
+ sqlite3* db;
10
+ std::string db_path;
11
+
12
+ bool create_tables();
13
+
14
+ public:
15
+ Database(const std::string& path = "cactus.db");
16
+ ~Database();
17
+
18
+ bool initialize();
19
+ bool register_app(const std::string& device_id, const std::string& token_expiry_date, const std::string& token);
20
+ bool get_registration(std::string& device_id, std::string& token_expiry_date, std::string& token);
21
+ const char* get_device_id();
22
+ void close();
23
+
24
+ bool is_open() const;
25
+ };
26
+
27
+ #endif // DATABASE_H
@@ -7,7 +7,7 @@ extern "C" {
7
7
 
8
8
  const char* register_app(const char* encrypted_data);
9
9
 
10
- const char* get_device_id();
10
+ const char* get_device_id(const char* current_token);
11
11
 
12
12
  // Helper function to free memory allocated by register_app
13
13
  void free_string(const char* str);
@@ -0,0 +1,27 @@
1
+ #ifndef DATABASE_H
2
+ #define DATABASE_H
3
+
4
+ #include <string>
5
+ struct sqlite3;
6
+
7
+ class Database {
8
+ private:
9
+ sqlite3* db;
10
+ std::string db_path;
11
+
12
+ bool create_tables();
13
+
14
+ public:
15
+ Database(const std::string& path = "cactus.db");
16
+ ~Database();
17
+
18
+ bool initialize();
19
+ bool register_app(const std::string& device_id, const std::string& token_expiry_date, const std::string& token);
20
+ bool get_registration(std::string& device_id, std::string& token_expiry_date, std::string& token);
21
+ const char* get_device_id();
22
+ void close();
23
+
24
+ bool is_open() const;
25
+ };
26
+
27
+ #endif // DATABASE_H