cui-llama.rn 1.7.4 → 1.7.6

package/cpp/tools/mtmd/mtmd.h

@@ -3,7 +3,6 @@
 
 #include "ggml.h"
 #include "llama.h"
-#include "clip.h"
 
 #include <stddef.h>
 #include <stdint.h>
@@ -109,6 +108,10 @@ MTMD_API bool mtmd_support_vision(mtmd_context * ctx);
 // whether the current model supports audio input
 MTMD_API bool mtmd_support_audio(mtmd_context * ctx);
 
+// get audio bitrate in Hz, for example 16000 for Whisper
+// return -1 if audio is not supported
+MTMD_API int mtmd_get_audio_bitrate(mtmd_context * ctx);
+
 // mtmd_bitmap
 //
 // if bitmap is image:
@@ -203,79 +206,12 @@ MTMD_API int32_t mtmd_encode_chunk(mtmd_context * ctx,
                                    const mtmd_input_chunk * chunk);
 
 // get output embeddings from the last encode pass
+// the reading size (in bytes) is equal to:
+// llama_model_n_embd(model) * mtmd_input_chunk_get_n_tokens(chunk) * sizeof(float)
 MTMD_API float * mtmd_get_output_embd(mtmd_context * ctx);
 
 /////////////////////////////////////////
 
-//
-// Helper functions (can be implemented based on other functions)
-//
-// Please note that these helpers are not guaranteed to be stable.
-// BREAKING CHANGES are expected.
-//
-
-// helper function to construct a mtmd_bitmap from a file
-// it calls mtmd_helper_bitmap_init_from_buf() internally
-// returns nullptr on failure
-// this function is thread-safe
-MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_file(const char * fname);
-
-// helper function to construct a mtmd_bitmap from a buffer containing a file
-// supported formats:
-//     image: formats supported by stb_image: jpg, png, bmp, gif, etc.
-//     audio: formats supported by miniaudio: wav, mp3, flac
-// note: audio files will be auto-detected based on magic bytes
-// returns nullptr on failure
-// this function is thread-safe
-MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(const unsigned char * buf, size_t len);
-
-// helper to count the total number of tokens from a list of chunks, useful to keep track of KV cache
-MTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks);
-
-// helper to count the total position of tokens from a list of chunks, useful to keep track of n_past
-// normally, n_pos is equal to n_tokens, but for M-RoPE it is different
-MTMD_API llama_pos mtmd_helper_get_n_pos(const mtmd_input_chunks * chunks);
-
-// helper function that automatically:
-// 1. run llama_decode() on text chunks
-// 2. run mtmd_encode() on image chunks, then mtmd_get_output_embd() and then llama_decode()
-// if any of the mtmd_encode() or llama_decode() calls return non-zero, stop and forward the error
-// otherwise, returns 0 on success
-// this function is NOT thread-safe
-MTMD_API int32_t mtmd_helper_eval_chunks(mtmd_context * ctx,
-                                         struct llama_context * lctx,
-                                         const mtmd_input_chunks * chunks,
-                                         llama_pos n_past,
-                                         llama_seq_id seq_id,
-                                         int32_t n_batch,
-                                         bool logits_last,
-                                         llama_pos * new_n_past);
-
-// works like mtmd_helper_eval_chunks(), but only for a single chunk
-// this function is NOT thread-safe
-MTMD_API int32_t mtmd_helper_eval_chunk_single(mtmd_context * ctx,
-                                               struct llama_context * lctx,
-                                               const mtmd_input_chunk * chunk,
-                                               llama_pos n_past,
-                                               llama_seq_id seq_id,
-                                               int32_t n_batch,
-                                               bool logits_last,
-                                               llama_pos * new_n_past);
-
-// helper function to decode an image whose embeddings have already been calculated
-// this helper will handle batching and pre/post decoding setup (for ex. gemma 3 requires non-causal attention)
-// ret 0 on success, -1 on chunk not being a valid image chunk, 1 on decode failure
-MTMD_API int32_t mtmd_helper_decode_image_chunk(mtmd_context * ctx,
-                                                struct llama_context * lctx,
-                                                const mtmd_input_chunk * chunk,
-                                                float * encoded_embd,
-                                                llama_pos n_past,
-                                                llama_seq_id seq_id,
-                                                int32_t n_batch,
-                                                llama_pos * new_n_past);
-
-/////////////////////////////////////////
-
 // test function, to be used in test-mtmd-c-api.c
 MTMD_API mtmd_input_chunks * mtmd_test_create_input_chunks(void);
 

package/cpp/unicode.cpp

@@ -204,12 +204,17 @@ static inline std::wstring unicode_wstring_from_utf8(const std::string & s) {
     // disable C++17 deprecation warning for std::codecvt_utf8
 #    pragma clang diagnostic push
 #    pragma clang diagnostic ignored "-Wdeprecated-declarations"
+#elif defined(__GNUC__)
+#    pragma GCC diagnostic push
+#    pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 #endif
 
     std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;
 
 #if defined(__clang__)
 #    pragma clang diagnostic pop
+#elif defined(__GNUC__)
+#    pragma GCC diagnostic pop
 #endif
 
     return conv.from_bytes(s);

package/ios/CMakeLists.txt

@@ -24,8 +24,19 @@ add_definitions(
     -DLM_GGML_METAL_USE_BF16
 )
 
+if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64;x86_64")
+    add_definitions(-DLM_GGML_CPU_GENERIC)
+endif ()
+
 set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../cpp)
 
+if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64")
+    set(SOURCE_FILES_ARCH
+        ${SOURCE_DIR}/ggml-cpu/arch/arm/quants.c
+        ${SOURCE_DIR}/ggml-cpu/arch/arm/repack.cpp
+    )
+endif ()
+
 # Define public headers
 set(PUBLIC_HEADERS
     ${SOURCE_DIR}/rn-llama.h
@@ -44,12 +55,11 @@ add_library(rnllama SHARED
     ${SOURCE_DIR}/ggml-cpu/amx/mmq.cpp
     ${SOURCE_DIR}/ggml-cpu/ggml-cpu.c
     ${SOURCE_DIR}/ggml-cpu/ggml-cpu.cpp
-    ${SOURCE_DIR}/ggml-cpu/ggml-cpu-aarch64.cpp
-    ${SOURCE_DIR}/ggml-cpu/ggml-cpu-quants.c
-    ${SOURCE_DIR}/ggml-cpu/ggml-cpu-traits.cpp
+    ${SOURCE_DIR}/ggml-cpu/quants.c
+    ${SOURCE_DIR}/ggml-cpu/traits.cpp
+    ${SOURCE_DIR}/ggml-cpu/repack.cpp
     ${SOURCE_DIR}/ggml-cpu/unary-ops.cpp
     ${SOURCE_DIR}/ggml-cpu/binary-ops.cpp
-    ${SOURCE_DIR}/ggml-cpu/sgemm.cpp
     ${SOURCE_DIR}/ggml-cpu/vec.cpp
     ${SOURCE_DIR}/ggml-cpu/ops.cpp
     ${SOURCE_DIR}/ggml-metal.m
@@ -65,7 +75,6 @@ add_library(rnllama SHARED
     ${SOURCE_DIR}/llama-adapter.cpp
     ${SOURCE_DIR}/llama-chat.cpp
     ${SOURCE_DIR}/llama-context.cpp
-    ${SOURCE_DIR}/llama-kv-cache.cpp
     ${SOURCE_DIR}/llama-arch.cpp
     ${SOURCE_DIR}/llama-batch.cpp
     ${SOURCE_DIR}/llama-cparams.cpp
@@ -75,6 +84,10 @@ add_library(rnllama SHARED
     ${SOURCE_DIR}/llama-model-loader.cpp
     ${SOURCE_DIR}/llama-model-saver.cpp
     ${SOURCE_DIR}/llama-mmap.cpp
+    ${SOURCE_DIR}/llama-kv-cache-unified.cpp
+    ${SOURCE_DIR}/llama-kv-cache-unified-iswa.cpp
+    ${SOURCE_DIR}/llama-memory-hybrid.cpp
+    ${SOURCE_DIR}/llama-memory-recurrent.cpp
     ${SOURCE_DIR}/llama-vocab.cpp
     ${SOURCE_DIR}/llama-memory.cpp
     ${SOURCE_DIR}/llama-io.cpp
@@ -87,13 +100,18 @@ add_library(rnllama SHARED
     ${SOURCE_DIR}/json-schema-to-grammar.cpp
     ${SOURCE_DIR}/minja/minja.hpp
     ${SOURCE_DIR}/minja/chat-template.hpp
-    ${SOURCE_DIR}/json.hpp
+    ${SOURCE_DIR}/nlohmann/json.hpp
+    ${SOURCE_DIR}/nlohmann/json_fwd.hpp
+    ${SOURCE_DIR}/chat-parser.cpp
+    ${SOURCE_DIR}/json-partial.cpp
+    ${SOURCE_DIR}/regex-partial.cpp
     # Multimodal support
     ${SOURCE_DIR}/tools/mtmd/mtmd.cpp
     ${SOURCE_DIR}/tools/mtmd/mtmd-audio.cpp
     ${SOURCE_DIR}/tools/mtmd/clip.cpp
     ${SOURCE_DIR}/tools/mtmd/mtmd-helper.cpp
     ${SOURCE_DIR}/rn-llama.cpp
+    ${SOURCE_FILES_ARCH}
 )
 
 # Setup include directories
@@ -102,6 +120,8 @@ target_include_directories(rnllama
         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../cpp>
         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../cpp/ggml-cpu>
         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../cpp/tools/mtmd>
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../cpp/minja>
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../cpp/nlohmann>
         $<INSTALL_INTERFACE:include>
 )
 

package/ios/RNLlama.h

@@ -4,7 +4,7 @@
 #if RNLLAMA_BUILD_FROM_SOURCE
 #import "json.hpp"
 #else
-#import <rnllama/json.hpp>
+#import <rnllama/nlohmann/json.hpp>
 #endif
 
 // TODO: Use RNLlamaSpec (Need to refactor NSDictionary usage)

package/ios/RNLlama.mm

@@ -102,13 +102,21 @@ RCT_EXPORT_METHOD(getFormattedChat:(double)contextId
         if ([params[@"jinja"] boolValue]) {
             NSString *jsonSchema = params[@"json_schema"];
             NSString *tools = params[@"tools"];
-            bool parallelToolCalls = [params[@"parallel_tool_calls"] boolValue];
+            BOOL parallelToolCalls = [params[@"parallel_tool_calls"] boolValue];
             NSString *toolChoice = params[@"tool_choice"];
-            resolve([context getFormattedChatWithJinja:messages withChatTemplate:chatTemplate withJsonSchema:jsonSchema withTools:tools withParallelToolCalls:parallelToolCalls withToolChoice:toolChoice]);
+            BOOL enableThinking = [params[@"enable_thinking"] boolValue];
+            resolve([context getFormattedChatWithJinja:messages
+                withChatTemplate:chatTemplate
+                withJsonSchema:jsonSchema
+                withTools:tools
+                withParallelToolCalls:parallelToolCalls
+                withToolChoice:toolChoice
+                withEnableThinking:enableThinking
+            ]);
         } else {
             resolve([context getFormattedChat:messages withChatTemplate:chatTemplate]);
         }
-    } catch (const nlohmann::json_abi_v3_11_3::detail::parse_error& e) {
+    } catch (const nlohmann::json_abi_v3_12_0::detail::parse_error& e) {
         NSString *errorMessage = [NSString stringWithUTF8String:e.what()];
         reject(@"llama_error", [NSString stringWithFormat:@"JSON parse error in getFormattedChat: %@", errorMessage], nil);
     } catch (const std::exception& e) { // catch cpp exceptions
@@ -297,6 +305,25 @@ RCT_EXPORT_METHOD(embedding:(double)contextId
     }
 }
 
+RCT_EXPORT_METHOD(rerank:(double)contextId
+                  query:(NSString *)query
+                  documents:(NSArray<NSString *> *)documents
+                  params:(NSDictionary *)params
+                  resolver:(RCTPromiseResolveBlock)resolve
+                  rejecter:(RCTPromiseRejectBlock)reject) {
+  RNLlamaContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+  if (context == nil) {
+    reject(@"context_not_found", @"Context not found", nil);
+    return;
+  }
+  @try {
+    NSArray *result = [context rerank:query documents:documents params:params];
+    resolve(result);
+  } @catch (NSException *exception) {
+    reject(@"rerank_error", exception.reason, nil);
+  }
+}
+
 RCT_EXPORT_METHOD(bench:(double)contextId
                   pp:(int)pp
                   tg:(int)tg
@@ -434,6 +461,129 @@ RCT_EXPORT_METHOD(releaseMultimodal:(double)contextId
     resolve(nil);
 }
 
+RCT_EXPORT_METHOD(initVocoder:(double)contextId
+                 withVocoderModelPath:(NSString *)vocoderModelPath
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    RNLlamaContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+    if ([context isPredicting]) {
+        reject(@"llama_error", @"Context is busy", nil);
+        return;
+    }
+
+    @try {
+        bool success = [context initVocoder:vocoderModelPath];
+        resolve(@(success));
+    } @catch (NSException *exception) {
+        reject(@"llama_cpp_error", exception.reason, nil);
+    }
+}
+
+RCT_EXPORT_METHOD(isVocoderEnabled:(double)contextId
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    RNLlamaContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+
+    resolve(@([context isVocoderEnabled]));
+}
+
+RCT_EXPORT_METHOD(getFormattedAudioCompletion:(double)contextId
+                 withSpeakerJsonStr:(NSString *)speakerJsonStr
+                 withTextToSpeak:(NSString *)textToSpeak
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    RNLlamaContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+
+    if (![context isVocoderEnabled]) {
+        reject(@"llama_error", @"Vocoder is not enabled", nil);
+        return;
+    }
+
+    @try {
+        NSString *result = [context getFormattedAudioCompletion:speakerJsonStr textToSpeak:textToSpeak];
+        resolve(result);
+    } @catch (NSException *exception) {
+        reject(@"llama_cpp_error", exception.reason, nil);
+    }
+}
+
+RCT_EXPORT_METHOD(getAudioCompletionGuideTokens:(double)contextId
+                 withTextToSpeak:(NSString *)textToSpeak
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    RNLlamaContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+
+    if (![context isVocoderEnabled]) {
+        reject(@"llama_error", @"Vocoder is not enabled", nil);
+        return;
+    }
+
+    @try {
+        NSArray *guideTokens = [context getAudioCompletionGuideTokens:textToSpeak];
+        resolve(guideTokens);
+    } @catch (NSException *exception) {
+        reject(@"llama_cpp_error", exception.reason, nil);
+    }
+}
+
+RCT_EXPORT_METHOD(decodeAudioTokens:(double)contextId
+                 withTokens:(NSArray *)tokens
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    RNLlamaContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+
+    if (![context isVocoderEnabled]) {
+        reject(@"llama_error", @"Vocoder is not enabled", nil);
+        return;
+    }
+
+    @try {
+        NSArray *audioData = [context decodeAudioTokens:tokens];
+        resolve(audioData);
+    } @catch (NSException *exception) {
+        reject(@"llama_cpp_error", exception.reason, nil);
+    }
+}
+
+RCT_EXPORT_METHOD(releaseVocoder:(double)contextId
+                 withResolver:(RCTPromiseResolveBlock)resolve
+                 withRejecter:(RCTPromiseRejectBlock)reject)
+{
+    RNLlamaContext *context = llamaContexts[[NSNumber numberWithDouble:contextId]];
+    if (context == nil) {
+        reject(@"llama_error", @"Context not found", nil);
+        return;
+    }
+
+    [context releaseVocoder];
+    resolve(nil);
+}
+
 RCT_EXPORT_METHOD(releaseContext:(double)contextId
                  withResolver:(RCTPromiseResolveBlock)resolve
                  withRejecter:(RCTPromiseRejectBlock)reject)

package/ios/RNLlamaContext.h

@@ -43,12 +43,14 @@
 - (NSDictionary *)tokenize:(NSString *)text imagePaths:(NSArray *)imagePaths;
 - (NSString *)detokenize:(NSArray *)tokens;
 - (NSDictionary *)embedding:(NSString *)text params:(NSDictionary *)params;
+- (NSArray *)rerank:(NSString *)query documents:(NSArray<NSString *> *)documents params:(NSDictionary *)params;
 - (NSDictionary *)getFormattedChatWithJinja:(NSString *)messages
     withChatTemplate:(NSString *)chatTemplate
     withJsonSchema:(NSString *)jsonSchema
     withTools:(NSString *)tools
     withParallelToolCalls:(BOOL)parallelToolCalls
-    withToolChoice:(NSString *)toolChoice;
+    withToolChoice:(NSString *)toolChoice
+    withEnableThinking:(BOOL)enableThinking;
 - (NSString *)getFormattedChat:(NSString *)messages withChatTemplate:(NSString *)chatTemplate;
 - (NSDictionary *)loadSession:(NSString *)path;
 - (int)saveSession:(NSString *)path size:(int)size;
@@ -56,6 +58,12 @@
 - (void)applyLoraAdapters:(NSArray *)loraAdapters;
 - (void)removeLoraAdapters;
 - (NSArray *)getLoadedLoraAdapters;
+- (bool)initVocoder:(NSString *)vocoderModelPath;
+- (bool)isVocoderEnabled;
+- (NSString *)getFormattedAudioCompletion:(NSString *)speakerJsonStr textToSpeak:(NSString *)textToSpeak;
+- (NSArray *)getAudioCompletionGuideTokens:(NSString *)textToSpeak;
+- (NSArray *)decodeAudioTokens:(NSArray *)tokens;
+- (void)releaseVocoder;
 - (void)invalidate;
 
 @end

package/ios/RNLlamaContext.mm

@@ -90,13 +90,6 @@
         NSLog(@"chatTemplate: %@", chatTemplate);
     }
 
-    NSString *reasoningFormat = params[@"reasoning_format"];
-    if (reasoningFormat && [reasoningFormat isEqualToString:@"deepseek"]) {
-        defaultParams.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
-    } else {
-        defaultParams.reasoning_format = COMMON_REASONING_FORMAT_NONE;
-    }
-
     if (params[@"n_ctx"]) defaultParams.n_ctx = [params[@"n_ctx"] intValue];
     if (params[@"use_mlock"]) defaultParams.use_mlock = [params[@"use_mlock"]boolValue];
 
@@ -362,6 +355,7 @@
     withTools:(NSString *)tools
     withParallelToolCalls:(BOOL)parallelToolCalls
     withToolChoice:(NSString *)toolChoice
+    withEnableThinking:(BOOL)enableThinking
 {
     auto tmpl_str = chatTemplate == nil ? "" : [chatTemplate UTF8String];
 
@@ -372,7 +366,8 @@
         jsonSchema == nil ? "" : [jsonSchema UTF8String],
         tools == nil ? "" : [tools UTF8String],
         parallelToolCalls,
-        toolChoice == nil ? "" : [toolChoice UTF8String]
+        toolChoice == nil ? "" : [toolChoice UTF8String],
+        enableThinking
     );
     result[@"prompt"] = [NSString stringWithUTF8String:chatParams.prompt.c_str()];
     result[@"chat_format"] = @(static_cast<int>(chatParams.format));
@@ -386,6 +381,7 @@
             @"token": @(trigger.token),
         }];
     }
+    result[@"thinking_forced_open"] = @(chatParams.thinking_forced_open);
     result[@"grammar_triggers"] = grammar_triggers;
     NSMutableArray *preserved_tokens = [[NSMutableArray alloc] init];
     for (const auto & token : chatParams.preserved_tokens) {
@@ -581,6 +577,16 @@
         }
     }
 
+    if (params[@"guide_tokens"] && [params[@"guide_tokens"] isKindOfClass:[NSArray class]]) {
+        NSArray *guide_tokens_array = params[@"guide_tokens"];
+        std::vector<llama_token> guide_tokens;
+        guide_tokens.reserve([guide_tokens_array count]);
+        for (NSNumber *token_num in guide_tokens_array) {
+            guide_tokens.push_back([token_num intValue]);
+        }
+        llama->setGuideTokens(guide_tokens);
+    }
+
     if (!llama->initSampling()) {
         @throw [NSException exceptionWithName:@"LlamaException" reason:@"Failed to initialize sampling" userInfo:nil];
     }
@@ -604,6 +610,9 @@
     } catch (const std::exception &e) {
         llama->endCompletion();
         @throw [NSException exceptionWithName:@"LlamaException" reason:[NSString stringWithUTF8String:e.what()] userInfo:nil];
+    } catch (const std::runtime_error& e) {
+        llama->endCompletion();
+        @throw [NSException exceptionWithName:@"LlamaException" reason:[NSString stringWithUTF8String:e.what()] userInfo:nil];
     }
 
     if (llama->context_full) {
@@ -680,7 +689,20 @@
     if (!llama->is_interrupted) {
         try {
             auto chat_format = params[@"chat_format"] ? [params[@"chat_format"] intValue] : COMMON_CHAT_FORMAT_CONTENT_ONLY;
-            common_chat_msg message = common_chat_parse(llama->generated_text, static_cast<common_chat_format>(chat_format));
+            common_chat_syntax chat_syntax;
+            chat_syntax.format = static_cast<common_chat_format>(chat_format);
+
+            NSString *reasoningFormat = params[@"reasoning_format"];
+            if (reasoningFormat && [reasoningFormat isEqualToString:@"deepseek"]) {
+                chat_syntax.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
+            } else if (reasoningFormat && [reasoningFormat isEqualToString:@"deepseek-legacy"]) {
+                chat_syntax.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY;
+            } else {
+                chat_syntax.reasoning_format = COMMON_REASONING_FORMAT_NONE;
+            }
+            chat_syntax.thinking_forced_open = [params[@"thinking_forced_open"] boolValue];
+
+            common_chat_msg message = common_chat_parse(llama->generated_text, false, chat_syntax);
             if (!message.reasoning_content.empty()) {
                 reasoningContent = [NSString stringWithUTF8String:message.reasoning_content.c_str()];
             }
@@ -716,6 +738,15 @@
     result[@"stopped_limit"] = @(llama->stopped_limit);
     result[@"stopping_word"] = [NSString stringWithUTF8String:llama->stopping_word.c_str()];
     result[@"tokens_cached"] = @(llama->n_past);
+
+    if (llama->isVocoderEnabled() && !llama->audio_tokens.empty()) {
+        NSMutableArray *audioTokens = [[NSMutableArray alloc] init];
+        for (llama_token token : llama->audio_tokens) {
+            [audioTokens addObject:@(token)];
+        }
+        result[@"audio_tokens"] = audioTokens;
+    }
+
     result[@"timings"] = @{
         @"prompt_n": @(timings.n_p_eval),
         @"prompt_ms": @(timings.t_p_eval_ms),
@@ -775,6 +806,8 @@
         return result;
     } catch (const std::exception &e) {
         @throw [NSException exceptionWithName:@"LlamaException" reason:[NSString stringWithUTF8String:e.what()] userInfo:nil];
+    } catch (const std::runtime_error& e) {
+        @throw [NSException exceptionWithName:@"LlamaException" reason:[NSString stringWithUTF8String:e.what()] userInfo:nil];
     }
 }
 
@@ -817,6 +850,9 @@
     } catch (const std::exception &e) {
       llama->endCompletion();
       @throw [NSException exceptionWithName:@"LlamaException" reason:[NSString stringWithUTF8String:e.what()] userInfo:nil];
+    } catch (const std::runtime_error& e) {
+      llama->endCompletion();
+      @throw [NSException exceptionWithName:@"LlamaException" reason:[NSString stringWithUTF8String:e.what()] userInfo:nil];
     }
     llama->doCompletion();
 
@@ -838,6 +874,34 @@
     return resultDict;
 }
 
+- (NSArray *)rerank:(NSString *)query documents:(NSArray<NSString *> *)documents params:(NSDictionary *)params {
+    // Convert NSArray to std::vector
+    std::vector<std::string> documentsVector;
+    for (NSString *doc in documents) {
+        documentsVector.push_back(std::string([doc UTF8String]));
+    }
+
+    NSMutableArray *resultArray = [[NSMutableArray alloc] init];
+
+    try {
+        std::vector<float> scores = llama->rerank(std::string([query UTF8String]), documentsVector);
+
+        // Create result array with score and index
+        for (size_t i = 0; i < scores.size(); i++) {
+            NSMutableDictionary *item = [[NSMutableDictionary alloc] init];
+            item[@"score"] = @(scores[i]);
+            item[@"index"] = @((int)i);
+            [resultArray addObject:item];
+        }
+    } catch (const std::exception &e) {
+        @throw [NSException exceptionWithName:@"LlamaException" reason:[NSString stringWithUTF8String:e.what()] userInfo:nil];
+    } catch (const std::runtime_error& e) {
+        @throw [NSException exceptionWithName:@"LlamaException" reason:[NSString stringWithUTF8String:e.what()] userInfo:nil];
+    }
+
+    return resultArray;
+}
+
 - (NSDictionary *)loadSession:(NSString *)path {
     if (!path || [path length] == 0) {
         @throw [NSException exceptionWithName:@"LlamaException" reason:@"Session path is empty" userInfo:nil];
@@ -920,6 +984,45 @@
     return result;
 }
 
+- (bool)initVocoder:(NSString *)vocoderModelPath {
+    return llama->initVocoder([vocoderModelPath UTF8String]);
+}
+
+- (bool)isVocoderEnabled {
+    return llama->isVocoderEnabled();
+}
+
+- (NSString *)getFormattedAudioCompletion:(NSString *)speakerJsonStr textToSpeak:(NSString *)textToSpeak {
+    std::string speakerStr = speakerJsonStr ? [speakerJsonStr UTF8String] : "";
+    return [NSString stringWithUTF8String:llama->getFormattedAudioCompletion(speakerStr, [textToSpeak UTF8String]).c_str()];
+}
+
+- (NSArray *)getAudioCompletionGuideTokens:(NSString *)textToSpeak {
+    std::vector<llama_token> guide_tokens = llama->getAudioCompletionGuideTokens([textToSpeak UTF8String]);
+    NSMutableArray *result = [[NSMutableArray alloc] init];
+    for (llama_token token : guide_tokens) {
+        [result addObject:@(token)];
+    }
+    return result;
+}
+
+- (NSArray *)decodeAudioTokens:(NSArray *)tokens {
+    std::vector<llama_token> token_vector;
+    for (NSNumber *token in tokens) {
+        token_vector.push_back([token intValue]);
+    }
+    std::vector<float> audio_data = llama->decodeAudioTokens(token_vector);
+    NSMutableArray *result = [[NSMutableArray alloc] init];
+    for (float sample : audio_data) {
+        [result addObject:@(sample)];
+    }
+    return result;
+}
+
+- (void)releaseVocoder {
+    llama->releaseVocoder();
+}
+
 - (void)invalidate {
     delete llama;
     // llama_backend_free();