cactus-react-native 0.0.1 → 0.1.1

package/android/src/newarch/java/com/cactus/CactusModule.java

@@ -78,40 +78,112 @@ public class CactusModule extends NativeCactusSpec {
   }
 
   @ReactMethod
-  public void tokenize(double id, final String text, final Promise promise) {
+  public void tokenize(double id, String text, Promise promise) {
     cactus.tokenize(id, text, promise);
   }
 
   @ReactMethod
-  public void detokenize(double id, final ReadableArray tokens, final Promise promise) {
+  public void tokenize(double id, String text, ReadableArray mediaPaths, Promise promise) {
+    cactus.tokenize(id, text, mediaPaths, promise);
+  }
+
+  @ReactMethod
+  public void detokenize(double id, ReadableArray tokens, Promise promise) {
     cactus.detokenize(id, tokens, promise);
   }
 
   @ReactMethod
-  public void embedding(double id, final String text, final ReadableMap params, final Promise promise) {
+  public void embedding(double id, String text, ReadableMap params, Promise promise) {
     cactus.embedding(id, text, params, promise);
   }
 
   @ReactMethod
-  public void bench(double id, final double pp, final double tg, final double pl, final double nr, final Promise promise) {
+  public void bench(double id, double pp, double tg, double pl, double nr, Promise promise) {
     cactus.bench(id, pp, tg, pl, nr, promise);
   }
 
   @ReactMethod
-  public void applyLoraAdapters(double id, final ReadableArray loraAdapters, final Promise promise) {
+  public void applyLoraAdapters(double id, ReadableArray loraAdapters, Promise promise) {
     cactus.applyLoraAdapters(id, loraAdapters, promise);
   }
 
   @ReactMethod
-  public void removeLoraAdapters(double id, final Promise promise) {
+  public void removeLoraAdapters(double id, Promise promise) {
     cactus.removeLoraAdapters(id, promise);
   }
 
   @ReactMethod
-  public void getLoadedLoraAdapters(double id, final Promise promise) {
+  public void getLoadedLoraAdapters(double id, Promise promise) {
     cactus.getLoadedLoraAdapters(id, promise);
   }
 
+  // New Multimodal Methods
+  @ReactMethod
+  public void initMultimodal(double id, String mmprojPath, Boolean useGpu, Promise promise) {
+    cactus.initMultimodal(id, mmprojPath, useGpu.booleanValue(), promise);
+  }
+
+  @ReactMethod
+  public void isMultimodalEnabled(double id, Promise promise) {
+    cactus.isMultimodalEnabled(id, promise);
+  }
+
+  @ReactMethod
+  public void isMultimodalSupportVision(double id, Promise promise) {
+    cactus.isMultimodalSupportVision(id, promise);
+  }
+
+  @ReactMethod
+  public void isMultimodalSupportAudio(double id, Promise promise) {
+    cactus.isMultimodalSupportAudio(id, promise);
+  }
+
+  @ReactMethod
+  public void releaseMultimodal(double id, Promise promise) {
+    cactus.releaseMultimodal(id, promise);
+  }
+
+  @ReactMethod
+  public void multimodalCompletion(double id, String prompt, ReadableArray mediaPaths, ReadableMap params, Promise promise) {
+    cactus.multimodalCompletion(id, prompt, mediaPaths, params, promise);
+  }
+
+  // New TTS/Vocoder Methods
+  @ReactMethod
+  public void initVocoder(double id, String vocoderModelPath, Promise promise) {
+    cactus.initVocoder(id, vocoderModelPath, promise);
+  }
+
+  @ReactMethod
+  public void isVocoderEnabled(double id, Promise promise) {
+    cactus.isVocoderEnabled(id, promise);
+  }
+
+  @ReactMethod
+  public void getTTSType(double id, Promise promise) {
+    cactus.getTTSType(id, promise);
+  }
+
+  @ReactMethod
+  public void getFormattedAudioCompletion(double id, String speakerJsonStr, String textToSpeak, Promise promise) {
+    cactus.getFormattedAudioCompletion(id, speakerJsonStr, textToSpeak, promise);
+  }
+
+  @ReactMethod
+  public void getAudioCompletionGuideTokens(double id, String textToSpeak, Promise promise) {
+    cactus.getAudioCompletionGuideTokens(id, textToSpeak, promise);
+  }
+
+  @ReactMethod
+  public void decodeAudioTokens(double id, ReadableArray tokens, Promise promise) {
+    cactus.decodeAudioTokens(id, tokens, promise);
+  }
+
+  @ReactMethod
+  public void releaseVocoder(double id, Promise promise) {
+    cactus.releaseVocoder(id, promise);
+  }
+
   @ReactMethod
   public void releaseContext(double id, Promise promise) {
     cactus.releaseContext(id, promise);

package/android/src/oldarch/java/com/cactus/CactusModule.java

@@ -83,6 +83,11 @@ public class CactusModule extends ReactContextBaseJavaModule {
     cactus.tokenize(id, text, promise);
   }
 
+  @ReactMethod
+  public void tokenize(double id, final String text, final ReadableArray mediaPaths, final Promise promise) {
+    cactus.tokenize(id, text, mediaPaths, promise);
+  }
+
   @ReactMethod
   public void detokenize(double id, final ReadableArray tokens, final Promise promise) {
     cactus.detokenize(id, tokens, promise);
@@ -113,6 +118,71 @@ public class CactusModule extends ReactContextBaseJavaModule {
     cactus.getLoadedLoraAdapters(id, promise);
   }
 
+  @ReactMethod
+  public void initMultimodal(double id, final String mmprojPath, final boolean useGpu, final Promise promise) {
+    cactus.initMultimodal(id, mmprojPath, useGpu, promise);
+  }
+
+  @ReactMethod
+  public void isMultimodalEnabled(double id, final Promise promise) {
+    cactus.isMultimodalEnabled(id, promise);
+  }
+
+  @ReactMethod
+  public void isMultimodalSupportVision(double id, final Promise promise) {
+    cactus.isMultimodalSupportVision(id, promise);
+  }
+
+  @ReactMethod
+  public void isMultimodalSupportAudio(double id, final Promise promise) {
+    cactus.isMultimodalSupportAudio(id, promise);
+  }
+
+  @ReactMethod
+  public void releaseMultimodal(double id, final Promise promise) {
+    cactus.releaseMultimodal(id, promise);
+  }
+
+  @ReactMethod
+  public void multimodalCompletion(double id, final String prompt, final ReadableArray mediaPaths, final ReadableMap params, final Promise promise) {
+    cactus.multimodalCompletion(id, prompt, mediaPaths, params, promise);
+  }
+
+  @ReactMethod
+  public void initVocoder(double id, final String vocoderModelPath, final Promise promise) {
+    cactus.initVocoder(id, vocoderModelPath, promise);
+  }
+
+  @ReactMethod
+  public void isVocoderEnabled(double id, final Promise promise) {
+    cactus.isVocoderEnabled(id, promise);
+  }
+
+  @ReactMethod
+  public void getTTSType(double id, final Promise promise) {
+    cactus.getTTSType(id, promise);
+  }
+
+  @ReactMethod
+  public void getFormattedAudioCompletion(double id, final String speakerJsonStr, final String textToSpeak, final Promise promise) {
+    cactus.getFormattedAudioCompletion(id, speakerJsonStr, textToSpeak, promise);
+  }
+
+  @ReactMethod
+  public void getAudioCompletionGuideTokens(double id, final String textToSpeak, final Promise promise) {
+    cactus.getAudioCompletionGuideTokens(id, textToSpeak, promise);
+  }
+
+  @ReactMethod
+  public void decodeAudioTokens(double id, final ReadableArray tokens, final Promise promise) {
+    cactus.decodeAudioTokens(id, tokens, promise);
+  }
+
+  @ReactMethod
+  public void releaseVocoder(double id, final Promise promise) {
+    cactus.releaseVocoder(id, promise);
+  }
+
   @ReactMethod
   public void releaseContext(double id, Promise promise) {
     cactus.releaseContext(id, promise);

package/cactus-react-native.podspec

@@ -8,11 +8,8 @@ if ENV["CACTUS_DISABLE_METAL"] != "1" then
   base_compiler_flags += " -DLM_GGML_USE_METAL -DLM_GGML_METAL_USE_BF16" # -DLM_GGML_METAL_NDEBUG
 end
 
-# Use base_optimizer_flags = "" for debug builds
-# base_optimizer_flags = ""
 base_optimizer_flags = "-O3 -DNDEBUG"
 
-# We always use the local ios directory since we're now copying the files
 ios_path = 'ios'
 
 Pod::Spec.new do |s|

package/ios/CMakeLists.txt

@@ -24,6 +24,7 @@ set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../cactus)
 # Define public headers
 set(PUBLIC_HEADERS
     ${SOURCE_DIR}/cactus.h
+    ${SOURCE_DIR}/cactus_ffi.h
     ${SOURCE_DIR}/llama.h
     ${SOURCE_DIR}/llama-impl.h
     ${SOURCE_DIR}/ggml.h
@@ -31,60 +32,81 @@ set(PUBLIC_HEADERS
 
 # Create library target
 add_library(cactus SHARED
-    ${SOURCE_DIR}/ggml.c
-    ${SOURCE_DIR}/ggml-alloc.c
-    ${SOURCE_DIR}/ggml-backend.cpp
-    ${SOURCE_DIR}/ggml-backend-reg.cpp
-    ${SOURCE_DIR}/ggml-cpu.c
-    ${SOURCE_DIR}/ggml-cpu.cpp
-    ${SOURCE_DIR}/ggml-cpu-aarch64.cpp
-    ${SOURCE_DIR}/ggml-cpu-quants.c
-    ${SOURCE_DIR}/ggml-cpu-traits.cpp
-    ${SOURCE_DIR}/ggml-metal.m
-    ${SOURCE_DIR}/ggml-opt.cpp
-    ${SOURCE_DIR}/ggml-threading.cpp
-    ${SOURCE_DIR}/ggml-quants.c
-    ${SOURCE_DIR}/gguf.cpp
-    ${SOURCE_DIR}/log.cpp
-    ${SOURCE_DIR}/llama-impl.cpp
-    ${SOURCE_DIR}/llama-grammar.cpp
-    ${SOURCE_DIR}/llama-sampling.cpp
-    ${SOURCE_DIR}/llama-vocab.cpp
-    ${SOURCE_DIR}/llama-adapter.cpp
-    ${SOURCE_DIR}/llama-chat.cpp
-    ${SOURCE_DIR}/llama-context.cpp
-    ${SOURCE_DIR}/llama-kv-cache.cpp
-    ${SOURCE_DIR}/llama-arch.cpp
-    ${SOURCE_DIR}/llama-batch.cpp
+    ${SOURCE_DIR}/cactus_context.cpp
+    ${SOURCE_DIR}/cactus_loader.cpp
+    ${SOURCE_DIR}/cactus_completion.cpp
+    ${SOURCE_DIR}/cactus_utils.cpp
+    ${SOURCE_DIR}/cactus_embedding.cpp
+    ${SOURCE_DIR}/cactus_lora.cpp
+    ${SOURCE_DIR}/cactus_tokenization.cpp
+    ${SOURCE_DIR}/cactus_multimodal.cpp
+    ${SOURCE_DIR}/cactus_tts.cpp
+    ${SOURCE_DIR}/cactus_bench.cpp
+    ${SOURCE_DIR}/cactus_chat.cpp
+    ${SOURCE_DIR}/cactus_ffi.cpp
+    ${SOURCE_DIR}/tools/mtmd/mtmd.cpp
+    ${SOURCE_DIR}/tools/mtmd/mtmd-audio.cpp
+    ${SOURCE_DIR}/tools/mtmd/clip.cpp
+    ${SOURCE_DIR}/tools/mtmd/mtmd-helper.cpp
+    ${SOURCE_DIR}/llama.cpp
+    ${SOURCE_DIR}/llama-mmap.cpp
+    ${SOURCE_DIR}/llama-memory.cpp
+    ${SOURCE_DIR}/llama-io.cpp
     ${SOURCE_DIR}/llama-cparams.cpp
     ${SOURCE_DIR}/llama-hparams.cpp
-    ${SOURCE_DIR}/llama.cpp
     ${SOURCE_DIR}/llama-model.cpp
     ${SOURCE_DIR}/llama-model-loader.cpp
-    ${SOURCE_DIR}/llama-mmap.cpp
+    ${SOURCE_DIR}/llama-model-saver.cpp
+    ${SOURCE_DIR}/llama-kv-cache.cpp
+    ${SOURCE_DIR}/llama-context.cpp
+    ${SOURCE_DIR}/llama-chat.cpp
+    ${SOURCE_DIR}/llama-batch.cpp
+    ${SOURCE_DIR}/llama-arch.cpp
+    ${SOURCE_DIR}/llama-adapter.cpp
+    ${SOURCE_DIR}/llama-sampling.cpp
+    ${SOURCE_DIR}/llama-grammar.cpp
     ${SOURCE_DIR}/llama-vocab.cpp
-    ${SOURCE_DIR}/llama-memory.cpp
-    ${SOURCE_DIR}/llama-io.cpp
+    ${SOURCE_DIR}/llama-impl.cpp
     ${SOURCE_DIR}/llama-graph.cpp
-    ${SOURCE_DIR}/sampling.cpp
-    ${SOURCE_DIR}/unicode-data.cpp
-    ${SOURCE_DIR}/unicode.cpp
-    ${SOURCE_DIR}/sgemm.cpp
+    ${SOURCE_DIR}/ggml.c
+    ${SOURCE_DIR}/ggml-alloc.c
+    ${SOURCE_DIR}/ggml-backend.cpp
+    ${SOURCE_DIR}/ggml-quants.c
+    ${SOURCE_DIR}/ggml-opt.cpp
+    ${SOURCE_DIR}/ggml-threading.cpp
+    ${SOURCE_DIR}/ggml-backend-reg.cpp
+    ${SOURCE_DIR}/gguf.cpp
     ${SOURCE_DIR}/common.cpp
     ${SOURCE_DIR}/chat.cpp
+    ${SOURCE_DIR}/log.cpp
+    ${SOURCE_DIR}/sampling.cpp
     ${SOURCE_DIR}/json-schema-to-grammar.cpp
+    ${SOURCE_DIR}/unicode.cpp
+    ${SOURCE_DIR}/unicode-data.cpp
+    ${SOURCE_DIR}/ggml-cpu/amx/amx.cpp
+    ${SOURCE_DIR}/ggml-cpu/amx/mmq.cpp
+    ${SOURCE_DIR}/ggml-cpu/ggml-cpu.c
+    ${SOURCE_DIR}/ggml-cpu/ggml-cpu.cpp
+    ${SOURCE_DIR}/ggml-cpu/ggml-cpu-aarch64.cpp
+    ${SOURCE_DIR}/ggml-cpu/ggml-cpu-quants.c
+    ${SOURCE_DIR}/ggml-cpu/ggml-cpu-traits.cpp
+    ${SOURCE_DIR}/ggml-cpu/unary-ops.cpp
+    ${SOURCE_DIR}/ggml-cpu/binary-ops.cpp
+    ${SOURCE_DIR}/ggml-cpu/sgemm.cpp
+    ${SOURCE_DIR}/ggml-cpu/vec.cpp
+    ${SOURCE_DIR}/ggml-cpu/ops.cpp
+    ${SOURCE_DIR}/ggml-metal.m
     ${SOURCE_DIR}/minja/minja.hpp
     ${SOURCE_DIR}/minja/chat-template.hpp
     ${SOURCE_DIR}/json.hpp
-    ${SOURCE_DIR}/amx/amx.cpp
-    ${SOURCE_DIR}/amx/mmq.cpp
-    ${SOURCE_DIR}/cactus.cpp
 )
 
 # Setup include directories, use the cactus directory
 target_include_directories(cactus
     PUBLIC
         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../cactus>
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../cactus/ggml-cpu>
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../cactus/tools/mtmd>
         $<INSTALL_INTERFACE:include>
 )
 

package/ios/Cactus.mm

@@ -237,9 +237,28 @@ RCT_EXPORT_METHOD(tokenize:(double)contextId
         reject(@"llama_error", @"Context not found", nil);
         return;
     }
-    NSMutableArray *tokens = [context tokenize:text];
+    NSArray *tokens = [context tokenize:text];
     resolve(@{ @"tokens": tokens });
-    [tokens release];
+}
+
+RCT_EXPORT_METHOD(tokenize:(double)contextId
+                  text:(NSString *)text
+                  mediaPaths:(NSArray *)mediaPaths
+                  withResolver:(RCTPromiseResolveBlock)resolve
+                  withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    CactusContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+    
+    if (mediaPaths && [mediaPaths count] > 0) {
+        resolve([context tokenize:text withMediaPaths:mediaPaths]);
+    } else {
+        NSArray *tokens = [context tokenize:text];
+        resolve(@{ @"tokens": tokens });
+    }
 }
 
 RCT_EXPORT_METHOD(detokenize:(double)contextId
@@ -342,6 +361,118 @@ RCT_EXPORT_METHOD(getLoadedLoraAdapters:(double)contextId
     resolve([context getLoadedLoraAdapters]);
 }
 
+RCT_EXPORT_METHOD(initMultimodal:(double)contextId
+                 mmprojPath:(NSString *)mmprojPath
+                 useGpu:(BOOL)useGpu
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    CactusContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+    @try {
+        BOOL result = [context initMultimodal:mmprojPath useGpu:useGpu];
+        resolve(@(result));
+    } @catch (NSException *exception) {
+        reject(@"llama_cpp_error", exception.reason, nil);
+    }
+}
+
+RCT_EXPORT_METHOD(isMultimodalEnabled:(double)contextId
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    CactusContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+    resolve(@([context isMultimodalEnabled]));
+}
+
+RCT_EXPORT_METHOD(isMultimodalSupportVision:(double)contextId
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    CactusContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+    resolve(@([context isMultimodalSupportVision]));
+}
+
+RCT_EXPORT_METHOD(isMultimodalSupportAudio:(double)contextId
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    CactusContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+    resolve(@([context isMultimodalSupportAudio]));
+}
+
+RCT_EXPORT_METHOD(releaseMultimodal:(double)contextId
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    CactusContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+    [context releaseMultimodal];
+    resolve(nil);
+}
+
+RCT_EXPORT_METHOD(multimodalCompletion:(double)contextId
+                 prompt:(NSString *)prompt
+                 mediaPaths:(NSArray *)mediaPaths
+                 completionParams:(NSDictionary *)completionParams
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    CactusContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+    if ([context isPredicting]) {
+        reject(@"llama_error", @"Context is busy", nil);
+        return;
+    }
+    dispatch_async(llamaDQueue, ^{
+        @try {
+            @autoreleasepool {
+                NSDictionary* completionResult = [context multimodalCompletion:prompt
+                    withMediaPaths:mediaPaths
+                    params:completionParams
+                    onToken:^(NSMutableDictionary *tokenResult) {
+                        if (![completionParams[@"emit_partial_completion"] boolValue]) return;
+                        dispatch_async(dispatch_get_main_queue(), ^{
+                            [self sendEventWithName:@"@Cactus_onToken"
+                                body:@{
+                                    @"contextId": [NSNumber numberWithDouble:contextId],
+                                    @"tokenResult": tokenResult
+                                }
+                            ];
+                            [tokenResult release];
+                        });
+                    }
+                ];
+                resolve(completionResult);
+            }
+        } @catch (NSException *exception) {
+            reject(@"llama_cpp_error", exception.reason, nil);
+            [context stopCompletion];
+        }
+    });
+}
+
 RCT_EXPORT_METHOD(releaseContext:(double)contextId
                  withResolver:(RCTPromiseResolveBlock)resolve
                  withRejecter:(RCTPromiseRejectBlock)reject)
@@ -368,6 +499,116 @@ RCT_EXPORT_METHOD(releaseAllContexts:(RCTPromiseResolveBlock)resolve
     resolve(nil);
 }
 
+// New TTS/Vocoder Methods
+RCT_EXPORT_METHOD(initVocoder:(double)contextId
+                 vocoderModelPath:(NSString *)vocoderModelPath
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    CactusContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+    @try {
+        BOOL result = [context initVocoder:vocoderModelPath];
+        resolve(@(result));
+    } @catch (NSException *exception) {
+        reject(@"llama_cpp_error", exception.reason, nil);
+    }
+}
+
+RCT_EXPORT_METHOD(isVocoderEnabled:(double)contextId
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    CactusContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+    resolve(@([context isVocoderEnabled]));
+}
+
+RCT_EXPORT_METHOD(getTTSType:(double)contextId
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    CactusContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+    resolve(@{ @"type": @([context getTTSType]) });
+}
+
+RCT_EXPORT_METHOD(getFormattedAudioCompletion:(double)contextId
+                 speakerJsonStr:(NSString *)speakerJsonStr
+                 textToSpeak:(NSString *)textToSpeak
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    CactusContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+    @try {
+        NSString *result = [context getFormattedAudioCompletion:speakerJsonStr textToSpeak:textToSpeak];
+        resolve(@{ @"formatted_prompt": result });
+    } @catch (NSException *exception) {
+        reject(@"llama_cpp_error", exception.reason, nil);
+    }
+}
+
+RCT_EXPORT_METHOD(getAudioCompletionGuideTokens:(double)contextId
+                 textToSpeak:(NSString *)textToSpeak
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    CactusContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+    @try {
+        NSArray *tokens = [context getAudioCompletionGuideTokens:textToSpeak];
+        resolve(@{ @"tokens": tokens });
+    } @catch (NSException *exception) {
+        reject(@"llama_cpp_error", exception.reason, nil);
+    }
+}
+
+RCT_EXPORT_METHOD(decodeAudioTokens:(double)contextId
+                 tokens:(NSArray *)tokens
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    CactusContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+    @try {
+        NSArray *audioData = [context decodeAudioTokens:tokens];
+        resolve(@{ @"audio_data": audioData });
+    } @catch (NSException *exception) {
+        reject(@"llama_cpp_error", exception.reason, nil);
+    }
+}
+
+RCT_EXPORT_METHOD(releaseVocoder:(double)contextId
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    CactusContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+    [context releaseVocoder];
+    resolve(nil);
+}
 
 - (void)invalidate {
     if (llamaContexts == nil) {

package/ios/CactusContext.h

@@ -37,6 +37,7 @@
 - (NSDictionary *)completion:(NSDictionary *)params onToken:(void (^)(NSMutableDictionary *tokenResult))onToken;
 - (void)stopCompletion;
 - (NSArray *)tokenize:(NSString *)text;
+- (NSArray *)tokenize:(NSString *)text withMediaPaths:(NSArray *)mediaPaths;
 - (NSString *)detokenize:(NSArray *)tokens;
 - (NSDictionary *)embedding:(NSString *)text params:(NSDictionary *)params;
 - (NSDictionary *)getFormattedChatWithJinja:(NSString *)messages
@@ -52,6 +53,27 @@
 - (void)applyLoraAdapters:(NSArray *)loraAdapters;
 - (void)removeLoraAdapters;
 - (NSArray *)getLoadedLoraAdapters;
+
+// New Multimodal Methods
+- (BOOL)initMultimodal:(NSString *)mmprojPath useGpu:(BOOL)useGpu;
+- (BOOL)isMultimodalEnabled;
+- (BOOL)isMultimodalSupportVision;
+- (BOOL)isMultimodalSupportAudio;
+- (void)releaseMultimodal;
+- (NSDictionary *)multimodalCompletion:(NSString *)prompt 
+    withMediaPaths:(NSArray *)mediaPaths 
+    params:(NSDictionary *)params 
+    onToken:(void (^)(NSMutableDictionary *tokenResult))onToken;
+
+// New TTS/Vocoder Methods
+- (BOOL)initVocoder:(NSString *)vocoderModelPath;
+- (BOOL)isVocoderEnabled;
+- (int)getTTSType;
+- (NSString *)getFormattedAudioCompletion:(NSString *)speakerJsonStr textToSpeak:(NSString *)textToSpeak;
+- (NSArray *)getAudioCompletionGuideTokens:(NSString *)textToSpeak;
+- (NSArray *)decodeAudioTokens:(NSArray *)tokens;
+- (void)releaseVocoder;
+
 - (void)invalidate;
 
 @end