llama-cpp-pro 0.2.2

package/cpp/cap-ios-bridge.h

@@ -0,0 +1,61 @@
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Core
+int64_t  llama_init_context(const char *model_path, const char *params_json);
+void     llama_release_context(int64_t context_id);
+bool     llama_toggle_native_log(bool enabled);
+char    *llama_run_completion(int64_t context_id, const char *params_json);
+void     llama_free_completion_result(char *result_json);
+void     llama_stop_completion(int64_t context_id);
+/** JSON object: {"embedding":[float,...]} — free with llama_free_completion_result */
+char    *llama_run_embedding_json(int64_t context_id, const char *text, const char *params_json);
+
+// Rerank — returns JSON [{score,index},...]; free with llama_free_completion_result
+char    *llama_rerank_json(int64_t context_id, const char *query, const char *docs_json);
+
+// Bench — returns raw "[desc,size,nparams,ppAvg,ppStd,tgAvg,tgStd]"; free with llama_free_completion_result
+char    *llama_bench(int64_t context_id, int32_t pp, int32_t tg, int32_t pl, int32_t nr);
+
+// Session management — returns JSON; free with llama_free_completion_result
+// NOTE: named llama_cap_* to avoid clashing with the deprecated bool-returning
+// llama_load_session_file / llama_save_session_file in llama.h (llama-context.cpp).
+char    *llama_cap_load_session_file(int64_t context_id, const char *filepath);
+int32_t  llama_cap_save_session_file(int64_t context_id, const char *filepath, int32_t max_tokens);
+
+// LoRA adapters
+int32_t  llama_apply_lora_adapters(int64_t context_id, const char *lora_adapters_json);
+void     llama_remove_lora_adapters(int64_t context_id);
+/** Returns JSON [{path,scale},...]; free with llama_free_completion_result */
+char    *llama_get_loaded_lora_adapters(int64_t context_id, const char *unused);
+
+// Multimodal
+int32_t  llama_init_multimodal(int64_t context_id, const char *mmproj_path, int32_t use_gpu);
+int32_t  llama_is_multimodal_enabled(int64_t context_id);
+/** Returns JSON {vision:bool,audio:bool}; free with llama_free_completion_result */
+char    *llama_get_multimodal_support(int64_t context_id, const char *unused);
+void     llama_release_multimodal(int64_t context_id);
+
+// TTS / Vocoder
+int32_t  llama_init_vocoder(int64_t context_id, const char *vocoder_path, int32_t n_batch);
+int32_t  llama_is_vocoder_enabled(int64_t context_id);
+/** Returns JSON {prompt,grammar?}; free with llama_free_completion_result */
+char    *llama_get_formatted_audio_completion(int64_t context_id, const char *speaker_json, const char *text);
+/** Returns JSON [token,...]; free with llama_free_completion_result */
+char    *llama_get_audio_completion_guide_tokens(int64_t context_id, const char *text);
+/** Returns JSON [float,...]; free with llama_free_completion_result */
+char    *llama_decode_audio_tokens(int64_t context_id, const char *tokens_json);
+void     llama_release_vocoder(int64_t context_id);
+
+// GPU info — returns JSON {gpu:bool,reasonNoGPU:string}; free with llama_free_completion_result
+char    *llama_get_context_gpu_info(int64_t context_id, const char *unused);
+
+#ifdef __cplusplus
+}
+#endif

package/cpp/cap-llama.cpp

@@ -0,0 +1,412 @@
+#include "cap-llama.h"
+#include "cap-tts.h"
+#include "cap-mtmd.hpp"
+#include "cap-completion.h"
+
+// Include multimodal support
+#include "tools/mtmd/mtmd.h"
+#include "tools/mtmd/mtmd-helper.h"
+#include "tools/mtmd/clip.h"
+
+namespace capllama {
+
+static const std::vector<lm_ggml_type> kv_cache_types = {
+    LM_GGML_TYPE_F32,
+    LM_GGML_TYPE_F16,
+    LM_GGML_TYPE_BF16,
+    LM_GGML_TYPE_Q8_0,
+    LM_GGML_TYPE_Q4_0,
+    LM_GGML_TYPE_Q4_1,
+    LM_GGML_TYPE_IQ4_NL,
+    LM_GGML_TYPE_Q5_0,
+    LM_GGML_TYPE_Q5_1,
+};
+
+lm_ggml_type kv_cache_type_from_str(const std::string & s) {
+    if (s.empty()) {
+        return LM_GGML_TYPE_F16; // Default to F16 if empty string
+    }
+
+    for (const auto & type : kv_cache_types) {
+        if (lm_ggml_type_name(type) == s) {
+            return type;
+        }
+    }
+
+    // Return default type instead of throwing to avoid crashes
+    return LM_GGML_TYPE_F16;
+}
+
+
+void log(const char *level, const char *function, int line,
+                       const char *format, ...)
+{
+    va_list args;
+    #if defined(__ANDROID__)
+        char prefix[256];
+        snprintf(prefix, sizeof(prefix), "%s:%d %s", function, line, format);
+
+        va_start(args, format);
+        android_LogPriority priority;
+        if (strcmp(level, "ERROR") == 0) {
+            priority = ANDROID_LOG_ERROR;
+        } else if (strcmp(level, "WARNING") == 0) {
+            priority = ANDROID_LOG_WARN;
+        } else if (strcmp(level, "INFO") == 0) {
+            priority = ANDROID_LOG_INFO;
+        } else {
+            priority = ANDROID_LOG_DEBUG;
+        }
+        __android_log_vprint(priority, "RNLlama", prefix, args);
+        va_end(args);
+    #else
+        printf("[%s] %s:%d ", level, function, line);
+        va_start(args, format);
+        vprintf(format, args);
+        va_end(args);
+        printf("\n");
+    #endif
+}
+
+static bool ends_with(const std::string &str, const std::string &suffix)
+{
+    return str.size() >= suffix.size() &&
+           0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix);
+}
+
+static size_t find_partial_stop_string(const std::string &stop,
+                                       const std::string &text)
+{
+    if (!text.empty() && !stop.empty())
+    {
+        const char text_last_char = text.back();
+        for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--)
+        {
+            if (stop[char_index] == text_last_char)
+            {
+                const std::string current_partial = stop.substr(0, char_index + 1);
+                if (ends_with(text, current_partial))
+                {
+                    return text.size() - char_index - 1;
+                }
+            }
+        }
+    }
+    return std::string::npos;
+}
+
+// format incomplete utf-8 multibyte character for output
+std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token)
+{
+    std::string out = token == -1 ? "" : common_token_to_piece(ctx, token);
+    // if the size is 1 and first bit is 1, meaning it's a partial character
+    //   (size > 1 meaning it's already a known token)
+    if (out.size() == 1 && (out[0] & 0x80) == 0x80)
+    {
+        std::stringstream ss;
+        ss << std::hex << (out[0] & 0xff);
+        std::string res(ss.str());
+        out = "byte: \\x" + res;
+    }
+    return out;
+}
+
+std::string tokens_to_str(llama_context *ctx, const std::vector<llama_token>::const_iterator begin, const std::vector<llama_token>::const_iterator end)
+{
+    std::string ret;
+    for (auto it = begin; it != end; ++it)
+    {
+        ret += common_token_to_piece(ctx, *it);
+    }
+    return ret;
+}
+
+
+llama_cap_context::~llama_cap_context() {
+    if (completion != nullptr) {
+        delete completion;
+        completion = nullptr;
+    }
+
+    releaseMultimodal();
+    releaseVocoder();
+    releaseDraftModel();  // Clean up speculative decoding resources
+}
+
+bool llama_cap_context::loadModel(common_params &params_)
+{
+    params = params_;
+#ifdef __EMSCRIPTEN__
+    fprintf(stderr, "@@WASM_LOAD@@ loadModel: common_init_from_params begin\n");
+#endif
+    llama_init = common_init_from_params(params);
+    model = llama_init.model.get();
+    ctx = llama_init.context.get();
+    if (model == nullptr)
+    {
+        LOG_ERROR("unable to load model: %s", params_.model.path.c_str());
+        return false;
+    }
+#ifdef __EMSCRIPTEN__
+    fprintf(stderr, "@@WASM_LOAD@@ loadModel: common_init_from_params ok, chat templates begin\n");
+#endif
+    templates = common_chat_templates_init(model, params.chat_template);
+#ifdef __EMSCRIPTEN__
+    fprintf(stderr, "@@WASM_LOAD@@ loadModel: chat templates ok, completion init begin\n");
+#endif
+    n_ctx = llama_n_ctx(ctx);
+
+    // Initialize completion context
+    if (completion != nullptr) {
+        delete completion;
+    }
+    completion = new llama_cap_context_completion(this);
+
+#ifdef __EMSCRIPTEN__
+    fprintf(stderr, "@@WASM_LOAD@@ loadModel: completion ok\n");
+#endif
+
+    // Initialize context shift flag
+    LOG_INFO("ctx_shift: %s", params.ctx_shift ? "enabled" : "disabled");
+
+    // We can uncomment for debugging or after this fix: https://github.com/ggerganov/llama.cpp/pull/11101
+    // LOG_INFO("%s\n", common_params_get_system_info(params).c_str());
+
+    return true;
+}
+
+
+bool llama_cap_context::validateModelChatTemplate(bool use_jinja, const char *name) const {
+    const char * tmpl = llama_model_chat_template(model, name);
+    if (tmpl == nullptr) {
+      return false;
+    }
+    return common_chat_verify_template(tmpl, use_jinja);
+}
+
+common_chat_params llama_cap_context::getFormattedChatWithJinja(
+        const std::string& messages,
+        const std::string& chat_template,
+        const std::string& json_schema,
+        const std::string& tools,
+        const bool& parallel_tool_calls,
+        const std::string& tool_choice,
+        const bool& enable_thinking,
+        const bool& add_generation_prompt,
+        const std::string& now_str,
+        const std::map<std::string, std::string>& chat_template_kwargs
+) const {
+    common_chat_templates_inputs inputs;
+    inputs.use_jinja = true;
+    inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(messages));
+    auto useTools = !tools.empty();
+    if (useTools) {
+        inputs.tools = common_chat_tools_parse_oaicompat(json::parse(tools));
+    }
+    inputs.parallel_tool_calls = parallel_tool_calls;
+    if (!tool_choice.empty()) {
+        inputs.tool_choice = common_chat_tool_choice_parse_oaicompat(tool_choice);
+    }
+    if (!json_schema.empty()) {
+        inputs.json_schema = json::parse(json_schema);
+    }
+    inputs.enable_thinking = enable_thinking;
+    inputs.add_generation_prompt = add_generation_prompt;
+
+    // Handle now parameter - parse timestamp or use current time
+    if (!now_str.empty()) {
+        try {
+            // Try to parse as timestamp (seconds since epoch)
+            auto timestamp = std::stoll(now_str);
+            inputs.now = std::chrono::system_clock::from_time_t(timestamp);
+        } catch (...) {
+            // If parsing fails, use current time
+            inputs.now = std::chrono::system_clock::now();
+        }
+    }
+
+    inputs.chat_template_kwargs = chat_template_kwargs;
+
+    // If chat_template is provided, create new one and use it (probably slow)
+    if (!chat_template.empty()) {
+        auto tmps = common_chat_templates_init(model, chat_template);
+        return common_chat_templates_apply(tmps.get(), inputs);
+    } else {
+        return common_chat_templates_apply(templates.get(), inputs);
+    }
+}
+
+std::string llama_cap_context::getFormattedChat(
+  const std::string &messages,
+  const std::string &chat_template
+) const {
+    common_chat_templates_inputs inputs;
+    inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(messages));
+    inputs.use_jinja = false;
+
+    // If chat_template is provided, create new one and use it (probably slow)
+    if (!chat_template.empty()) {
+        auto tmps = common_chat_templates_init(model, chat_template);
+        return common_chat_templates_apply(tmps.get(), inputs).prompt;
+    } else {
+        return common_chat_templates_apply(templates.get(), inputs).prompt;
+    }
+}
+
+llama_cap_tokenize_result llama_cap_context::tokenize(const std::string &text, const std::vector<std::string> &media_paths) {
+  if (media_paths.size() > 0) {
+      if (!isMultimodalEnabled()) {
+          throw std::runtime_error("Multimodal is not enabled but media paths are provided");
+      }
+      auto result = tokenizeWithMedia(mtmd_wrapper, text, media_paths);
+      mtmd_input_chunks_free(result.chunks);
+      llama_cap_tokenize_result tokenize_result = {
+          .tokens = result.tokens,
+          .has_media = true,
+          .bitmap_hashes = result.bitmap_hashes,
+          .chunk_pos = result.chunk_pos,
+          .chunk_pos_media = result.chunk_pos_media,
+      };
+      return tokenize_result;
+  }
+  std::vector<llama_token> text_tokens;
+  text_tokens = common_tokenize(ctx, text, false);
+  llama_cap_tokenize_result tokenize_result = {
+      .tokens = text_tokens,
+      .has_media = false,
+      .bitmap_hashes = {},
+      .chunk_pos = {},
+      .chunk_pos_media = {},
+  };
+  return tokenize_result;
+}
+
+int llama_cap_context::applyLoraAdapters(std::vector<common_adapter_lora_info> lora) {
+    for (auto &la : lora) {
+        la.ptr = llama_adapter_lora_init(model, la.path.c_str());
+        if (la.ptr == nullptr) {
+            LOG_ERROR("failed to apply lora adapter '%s'\n", la.path.c_str());
+            return -1;
+        }
+    }
+    this->lora = lora;
+    common_set_adapter_lora(ctx, lora);
+    return 0;
+}
+
+void llama_cap_context::removeLoraAdapters() {
+    this->lora.clear();
+    common_set_adapter_lora(ctx, this->lora); // apply empty list
+}
+
+std::vector<common_adapter_lora_info> llama_cap_context::getLoadedLoraAdapters() {
+    return this->lora;
+}
+
+bool llama_cap_context::initMultimodal(const std::string &mmproj_path, bool use_gpu) {
+    try {
+        mtmd_wrapper = new llama_cap_context_mtmd(mmproj_path, use_gpu, model, ctx, params, has_multimodal, params);
+        return true;
+    } catch (const std::exception& e) {
+        LOG_ERROR("[DEBUG] Failed to initialize multimodal: %s", e.what());
+        return false;
+    }
+}
+
+bool llama_cap_context::isMultimodalEnabled() const {
+    return mtmd_wrapper != nullptr && mtmd_wrapper->isEnabled(has_multimodal);
+}
+
+bool llama_cap_context::isMultimodalSupportVision() const {
+    return isMultimodalEnabled() && mtmd_wrapper->supportVision();
+}
+
+bool llama_cap_context::isMultimodalSupportAudio() const {
+    return isMultimodalEnabled() && mtmd_wrapper->supportAudio();
+}
+
+void llama_cap_context::releaseMultimodal() {
+    if (mtmd_wrapper != nullptr) {
+        delete mtmd_wrapper;
+        mtmd_wrapper = nullptr;
+        has_multimodal = false;
+    }
+}
+
+bool llama_cap_context::initVocoder(const std::string &vocoder_model_path, int batch_size) {
+    try {
+        tts_wrapper = new llama_cap_context_tts(vocoder_model_path, batch_size);
+        has_vocoder = true;
+        return true;
+    } catch (const std::exception& e) {
+        has_vocoder = false;
+        return false;
+    }
+}
+
+bool llama_cap_context::isVocoderEnabled() const {
+    return has_vocoder && tts_wrapper != nullptr;
+}
+
+void llama_cap_context::releaseVocoder() {
+    if (tts_wrapper != nullptr) {
+        delete tts_wrapper;
+        tts_wrapper = nullptr;
+    }
+    has_vocoder = false;
+}
+
+// Speculative decoding methods
+bool llama_cap_context::loadDraftModel(const std::string &draft_model_path) {
+    if (draft_model_path.empty()) {
+        return false;
+    }
+
+    // Create draft model parameters (based on main model params)
+    common_params draft_params = params;
+    draft_params.model.path = draft_model_path;
+    
+    // Mobile optimization: smaller context for draft model
+    if (mobile_speculative) {
+        draft_params.n_ctx = std::min(params.n_ctx, 1024);  // Limit draft context
+        draft_params.n_batch = std::min(params.n_batch, 128);  // Smaller batch
+    }
+
+    try {
+        // For now, use simplified draft model initialization
+        // This would be expanded in a full implementation to properly initialize
+        // the draft model and context
+        
+        // TODO: Implement proper draft model loading
+        // draft_model = llama_load_model_from_file(draft_model_path.c_str(), draft_params);
+        // draft_ctx = llama_new_context_with_model(draft_model, draft_params);
+        
+        // For this implementation, we'll disable speculative decoding
+        // until proper model loading is implemented
+        printf("Draft model loading not yet implemented - falling back to regular decoding\n");
+        speculative_enabled = false;
+        return false;
+        
+    } catch (const std::exception& e) {
+        printf("Failed to load draft model: %s\n", e.what());
+        releaseDraftModel();
+    }
+    
+    return false;
+}
+
+void llama_cap_context::releaseDraftModel() {
+    if (draft_ctx) {
+        // Note: draft_ctx and draft_model are managed by common_init_result
+        // They will be automatically cleaned up
+        draft_ctx = nullptr;
+        draft_model = nullptr;
+    }
+    speculative_enabled = false;
+}
+
+bool llama_cap_context::isSpectulativeEnabled() const {
+    return speculative_enabled && draft_model && draft_ctx;
+}
+
+}

package/cpp/cap-llama.h

@@ -0,0 +1,161 @@
+#ifndef CAPLLAMA_H
+#define CAPLLAMA_H
+
+#include <sstream>
+#include <iostream>
+#include <thread>
+#include <codecvt>
+#include "chat.h"
+#include "common.h"
+#include "ggml.h"
+#include "gguf.h"
+#include "llama.h"
+#include "llama-model.h"
+#include "llama-impl.h"
+#include "sampling.h"
+#include "nlohmann/json.hpp"
+#include "cap-tts.h"
+#if defined(__ANDROID__)
+#include <android/log.h>
+#endif
+
+using json = nlohmann::ordered_json;
+
+namespace capllama {
+
+std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token);
+
+std::string tokens_to_str(llama_context *ctx, const std::vector<llama_token>::const_iterator begin, const std::vector<llama_token>::const_iterator end);
+
+lm_ggml_type kv_cache_type_from_str(const std::string & s);
+
+// Forward declarations - actual definitions are in cap-completion.h
+// Note: enum forward declarations not allowed in C++, using include in implementation file
+struct completion_token_output;
+struct completion_partial_output;
+struct llama_cap_context_mtmd;
+
+struct llama_cap_context_tts;
+
+struct llama_cap_context_completion;
+
+struct llama_cap_tokenize_result {
+  std::vector<llama_token> tokens;
+  bool has_media = false;
+  std::vector<std::string> bitmap_hashes;
+  std::vector<size_t> chunk_pos; // both text and media
+  std::vector<size_t> chunk_pos_media; // media only
+};
+
+// Main context class
+struct llama_cap_context {
+    // Model state fields
+    llama_model *model = nullptr;
+    float loading_progress = 0;
+    bool is_load_interrupted = false;
+    common_params params;
+    common_init_result llama_init;
+    llama_context *ctx = nullptr;
+    common_chat_templates_ptr templates;
+    int n_ctx;
+
+    // Speculative decoding fields
+    llama_model *draft_model = nullptr;
+    llama_context *draft_ctx = nullptr;
+    bool speculative_enabled = false;
+    int speculative_samples = 3;  // Mobile-optimized default
+    bool mobile_speculative = true;
+
+    // Completion context
+    llama_cap_context_completion *completion = nullptr;
+
+    ~llama_cap_context();
+
+    bool loadModel(common_params &params_);
+
+    // Speculative decoding methods
+    bool loadDraftModel(const std::string &draft_model_path);
+    void releaseDraftModel();
+    bool isSpectulativeEnabled() const;
+
+    // Model methods
+    bool validateModelChatTemplate(bool use_jinja, const char *name) const;
+    common_chat_params getFormattedChatWithJinja(
+      const std::string& messages,
+      const std::string& chat_template,
+      const std::string& json_schema,
+      const std::string& tools,
+      const bool& parallel_tool_calls,
+      const std::string& tool_choice,
+      const bool& enable_thinking,
+      const bool& add_generation_prompt = true,
+      const std::string& now_str = "",
+      const std::map<std::string, std::string>& chat_template_kwargs = {}
+    ) const;
+    std::string getFormattedChat(
+      const std::string &messages,
+      const std::string &chat_template
+    ) const;
+    llama_cap_tokenize_result tokenize(const std::string &text, const std::vector<std::string> &media_paths);
+
+    // Lora methods
+    std::vector<common_adapter_lora_info> lora;
+    int applyLoraAdapters(std::vector<common_adapter_lora_info> lora);
+    void removeLoraAdapters();
+    std::vector<common_adapter_lora_info> getLoadedLoraAdapters();
+
+    // Multimodal fields and methods
+    llama_cap_context_mtmd *mtmd_wrapper = nullptr;
+    bool has_multimodal = false;
+    bool initMultimodal(const std::string &mmproj_path, bool use_gpu);
+    bool isMultimodalEnabled() const;
+    bool isMultimodalSupportVision() const;
+    bool isMultimodalSupportAudio() const;
+    void releaseMultimodal();
+
+    // TTS fields and methods (delegated to TTS context)
+    llama_cap_context_tts *tts_wrapper = nullptr;
+    bool has_vocoder = false;
+    bool initVocoder(const std::string &vocoder_model_path, int batch_size = -1);
+    bool isVocoderEnabled() const;
+    void releaseVocoder();
+};
+
+// Utility functions
+inline void llama_batch_add(llama_batch *batch, llama_token id, llama_pos pos, std::vector<llama_seq_id> seq_ids, bool logits) {
+    batch->token   [batch->n_tokens] = id;
+    batch->pos     [batch->n_tokens] = pos;
+    batch->n_seq_id[batch->n_tokens] = seq_ids.size();
+    for (size_t i = 0; i < seq_ids.size(); i++) {
+        batch->seq_id[batch->n_tokens][i] = seq_ids[i];
+    }
+    batch->logits  [batch->n_tokens] = logits ? 1 : 0;
+    batch->n_tokens += 1;
+}
+
+// Logging functions
+void log(const char *level, const char *function, int line, const char *format, ...);
+
+// Logging macros
+extern bool capllama_verbose;
+
+#if CAPLLAMA_VERBOSE != 1
+#define LOG_VERBOSE(MSG, ...)
+#else
+#define LOG_VERBOSE(MSG, ...)                                       \
+    do                                                              \
+    {                                                               \
+        if (capllama_verbose)                                        \
+        {                                                           \
+            log("VERBOSE", __func__, __LINE__, MSG, ##__VA_ARGS__); \
+        }                                                           \
+    } while (0)
+#endif
+
+#define LOG_ERROR(MSG, ...) log("ERROR", __func__, __LINE__, MSG, ##__VA_ARGS__)
+#define LOG_WARNING(MSG, ...) log("WARNING", __func__, __LINE__, MSG, ##__VA_ARGS__)
+#define LOG_INFO(MSG, ...) log("INFO", __func__, __LINE__, MSG, ##__VA_ARGS__)
+
+} // namespace capllama
+
+#endif /* CAPLLAMA_H */