llama-cpp-pro 0.2.2

package/cpp/cap-tts.h

@@ -0,0 +1,59 @@
+#ifndef CAPTTS_H
+#define CAPTTS_H
+
+#include <vector>
+#include <string>
+#include "llama.h"
+#include "nlohmann/json.hpp"
+#include "common.h"
+
+using json = nlohmann::ordered_json;
+
+namespace capllama {
+
+// Forward declarations
+struct llama_cap_context;
+
+// TTS type enumeration
+enum tts_type {
+    UNKNOWN = -1,
+    OUTETTS_V0_1 = 0,
+    OUTETTS_V0_2 = 1,
+    OUTETTS_V0_3 = 2,
+};
+
+// Audio completion result structure
+struct llama_cap_audio_completion_result {
+    std::string prompt;
+    const char *grammar;
+};
+
+// TTS context for TTS-specific functionality
+struct llama_cap_context_tts {
+    // TTS state fields
+    std::vector<llama_token> audio_tokens;
+    std::vector<llama_token> guide_tokens;
+    bool next_token_uses_guide_token = true;
+
+    // Vocoder fields (from llama_cap_context_vocoder)
+    common_init_result init_result;
+    common_params params;
+    llama_model *model = nullptr;
+    llama_context *ctx = nullptr;
+    tts_type type = UNKNOWN;
+
+    // Constructor and destructor
+    llama_cap_context_tts(const std::string &vocoder_model_path, int batch_size = -1);
+    ~llama_cap_context_tts();
+
+    // TTS utility methods
+    tts_type getTTSType(llama_cap_context* main_ctx, json speaker = nullptr);
+    llama_cap_audio_completion_result getFormattedAudioCompletion(llama_cap_context* main_ctx, const std::string &speaker_json_str, const std::string &text_to_speak);
+    std::vector<llama_token> getAudioCompletionGuideTokens(llama_cap_context* main_ctx, const std::string &text_to_speak);
+    std::vector<float> decodeAudioTokens(llama_cap_context* main_ctx, const std::vector<llama_token> &tokens);
+    void setGuideTokens(const std::vector<llama_token> &tokens);
+};
+
+}
+
+#endif /* CAPTTS_H */

package/cpp/cap-wasm-fs.cpp

@@ -0,0 +1,156 @@
+// JSPI async GGUF file read — intercepts libc fread when async mode is enabled.
+// Model bytes stay in JS (Blob / OPFS reader); WASM reads on demand in 1 MB chunks.
+// Ref: ref-code/wllama (wllama-fs.h)
+
+#if defined(CAPLLAMA_BUILD_WASM_JSPI) && defined(__EMSCRIPTEN__)
+
+#include <algorithm>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <map>
+#include <string>
+#include <vector>
+
+#include <emscripten.h>
+
+static std::map<FILE *, std::string> s_file_path_map;
+
+namespace cap_wasm_fs {
+
+// Set from JS via cap_wasm_set_use_async_file() — do not cache getenv once at startup
+// (getenv may run before Module.ENV is applied, locking async off permanently).
+static bool use_async_explicit = false;
+
+static const size_t CACHE_SIZE = 1024 * 1024; // 1 MB read-ahead
+
+std::vector<uint8_t> cache_data;
+size_t             cache_start = 0;
+FILE *             cache_file  = nullptr;
+
+bool use_async_file() {
+    if (use_async_explicit) {
+        return true;
+    }
+    return getenv("USE_ASYNC_FILE") != nullptr;
+}
+
+void set_use_async_file(bool enabled) {
+    use_async_explicit = enabled;
+}
+
+size_t try_cache(FILE *f, char *ptr, size_t req_bytes, size_t fpos) {
+    if (f != cache_file || cache_data.empty()) {
+        return 0;
+    }
+    if (fpos >= cache_start && fpos + req_bytes <= cache_start + cache_data.size()) {
+        memcpy(ptr, cache_data.data() + (fpos - cache_start), req_bytes);
+        return req_bytes;
+    }
+    return 0;
+}
+
+} // namespace cap_wasm_fs
+
+// Implemented in llama_engine.js shim (_cap_js_file_read) — sync EM_JS (no JSPI suspend).
+EM_JS(size_t, cap_js_file_read_impl, (const char *path_ptr, size_t offset, size_t req_size, void *out_ptr), {
+  return Number(_cap_js_file_read(UTF8ToString(Number(path_ptr)), Number(offset), Number(req_size), Number(out_ptr)));
+});
+
+extern "C" {
+
+void cap_wasm_set_use_async_file(int enabled) {
+    cap_wasm_fs::set_use_async_file(enabled != 0);
+}
+
+FILE *__real_fopen(const char *path, const char *mode);
+int   __real_fclose(FILE *f);
+size_t __real_fread(void *ptr, size_t size, size_t nmemb, FILE *f);
+int   __real_fseek(FILE *f, long offset, int whence);
+long  __real_ftell(FILE *f);
+
+FILE *__wrap_fopen(const char *path, const char *mode) {
+    FILE *f = __real_fopen(path, mode);
+    if (f) {
+        s_file_path_map[f] = path;
+    }
+    return f;
+}
+
+int __wrap_fclose(FILE *f) {
+    if (cap_wasm_fs::cache_file == f) {
+        cap_wasm_fs::cache_file = nullptr;
+        cap_wasm_fs::cache_data.clear();
+    }
+    s_file_path_map.erase(f);
+    return __real_fclose(f);
+}
+
+int __wrap_fseek(FILE *f, long offset, int whence) {
+    return __real_fseek(f, offset, whence);
+}
+
+long __wrap_ftell(FILE *f) {
+    return __real_ftell(f);
+}
+
+size_t __wrap_fread(void *ptr, size_t size, size_t nmemb, FILE *f) {
+    if (!cap_wasm_fs::use_async_file()) {
+        return __real_fread(ptr, size, nmemb, f);
+    }
+
+    auto nit = s_file_path_map.find(f);
+    if (nit == s_file_path_map.end()) {
+        return __real_fread(ptr, size, nmemb, f);
+    }
+
+    size_t req_bytes = size * nmemb;
+    if (req_bytes == 0) {
+        return 0;
+    }
+
+    size_t fpos = static_cast<size_t>(__real_ftell(f));
+
+    // Large reads (>= 1 MB): write directly into ptr, skip cache entirely.
+    if (req_bytes >= cap_wasm_fs::CACHE_SIZE) {
+        size_t actual = static_cast<size_t>(cap_js_file_read_impl(
+            nit->second.c_str(), fpos, req_bytes, ptr));
+        if (actual == 0) {
+            return 0;
+        }
+        size_t copy_bytes = std::min(req_bytes, actual);
+        __real_fseek(f, static_cast<long>(fpos + copy_bytes), SEEK_SET);
+        return copy_bytes / size;
+    }
+
+    // Small reads: try cache first.
+    size_t cached = cap_wasm_fs::try_cache(f, static_cast<char *>(ptr), req_bytes, fpos);
+    if (cached == req_bytes) {
+        __real_fseek(f, static_cast<long>(fpos + req_bytes), SEEK_SET);
+        return nmemb;
+    }
+
+    // Cache miss: fetch a full CACHE_SIZE block from JS.
+    cap_wasm_fs::cache_data.resize(cap_wasm_fs::CACHE_SIZE);
+    size_t actual = static_cast<size_t>(cap_js_file_read_impl(
+        nit->second.c_str(), fpos, cap_wasm_fs::CACHE_SIZE,
+        cap_wasm_fs::cache_data.data()));
+
+    cap_wasm_fs::cache_data.resize(actual);
+    cap_wasm_fs::cache_file  = f;
+    cap_wasm_fs::cache_start = fpos;
+
+    if (actual == 0) {
+        return 0;
+    }
+
+    size_t copy_bytes = std::min(req_bytes, actual);
+    memcpy(ptr, cap_wasm_fs::cache_data.data(), copy_bytes);
+    __real_fseek(f, static_cast<long>(fpos + copy_bytes), SEEK_SET);
+
+    return copy_bytes / size;
+}
+
+} // extern "C"
+
+#endif // CAPLLAMA_BUILD_WASM_JSPI && __EMSCRIPTEN__

package/cpp/cap-wasm-jspi.cpp

@@ -0,0 +1,19 @@
+#if defined(CAPLLAMA_BUILD_WASM_JSPI) && defined(__EMSCRIPTEN__)
+
+#include "cap-wasm-jspi.h"
+#include <emscripten/emscripten.h>
+
+// Dispatches each generated token to Module.__llamaStreamOnToken (set by wasm.engine.ts).
+EM_ASYNC_JS(void, cap_js_token_dispatch_impl, (const char *token_ptr, int index), {
+  if (Module.__llamaStreamOnToken) {
+    await Module.__llamaStreamOnToken(UTF8ToString(Number(token_ptr)), Number(index));
+  }
+});
+
+extern "C" void cap_wasm_jspi_token_dispatch(const char *token_text, int token_index) {
+    if (token_text) {
+        cap_js_token_dispatch_impl(token_text, token_index);
+    }
+}
+
+#endif

package/cpp/cap-wasm-jspi.h

@@ -0,0 +1,30 @@
+#pragma once
+
+// JSPI token streaming for Emscripten WASM builds.
+// Implementation lives in cap-wasm-jspi.cpp (EM_ASYNC_JS must not be in a
+// header included from other translation units — causes em_asm.h conflicts).
+
+#if defined(CAPLLAMA_BUILD_WASM_JSPI) && defined(__EMSCRIPTEN__)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void cap_wasm_jspi_token_dispatch(const char *token_text, int token_index);
+
+#ifdef __cplusplus
+}
+
+inline void cap_wasm_jspi_token_callback(
+    const char *token_text,
+    void * /*user_data*/,
+    int token_index)
+{
+    if (token_text && token_text[0] != '\0') {
+        cap_wasm_jspi_token_dispatch(token_text, token_index);
+    }
+}
+
+#endif // __cplusplus
+
+#endif // CAPLLAMA_BUILD_WASM_JSPI && __EMSCRIPTEN__

package/cpp/cap-wasm-vfs.cpp

@@ -0,0 +1,22 @@
+#if defined(CAPLLAMA_BUILD_WASM) && defined(__EMSCRIPTEN__)
+
+#include <emscripten.h>
+
+// Must live outside cap-ios-bridge.cpp's extern "C" block (emscripten.h uses C++ templates).
+extern "C" void cap_wasm_ensure_tmp_dir(void) {
+    EM_ASM({
+        if (typeof FS !== 'undefined') {
+            try {
+                if (!FS.analyzePath('/tmp').exists) {
+                    if (typeof FS.createPath === 'function') {
+                        FS.createPath('/', 'tmp', true, true);
+                    } else {
+                        FS.mkdir('/tmp');
+                    }
+                }
+            } catch (e) {}
+        }
+    });
+}
+
+#endif

package/cpp/chat-parser.cpp

@@ -0,0 +1,393 @@
+#include "chat-parser.h"
+#include "common.h"
+#include "log.h"
+#include "regex-partial.h"
+
+#include <optional>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+using json = nlohmann::ordered_json;
+
+common_chat_msg_parser::common_chat_msg_parser(const std::string & input, bool is_partial, const common_chat_syntax & syntax)
+    : input_(input), is_partial_(is_partial), syntax_(syntax)
+{
+    result_.role = "assistant";
+
+    while (true) {
+        std::string id = std::to_string(std::rand());
+        if (input.find(id) == std::string::npos) {
+            healing_marker_ = id;
+            break;
+        }
+    }
+}
+
+std::string common_chat_msg_parser::str(const common_string_range & rng) const {
+    LM_GGML_ASSERT(rng.begin <= rng.end);
+    return input_.substr(rng.begin, rng.end - rng.begin);
+}
+
+void common_chat_msg_parser::add_content(const std::string &content) {
+    result_.content += content;
+}
+
+void common_chat_msg_parser::add_reasoning_content(const std::string &reasoning_content) {
+    result_.reasoning_content += reasoning_content;
+}
+
+bool common_chat_msg_parser::add_tool_call(const std::string & name, const std::string & id, const std::string & arguments) {
+    if (name.empty()) {
+        return false;
+    }
+
+    common_chat_tool_call tool_call;
+    tool_call.name = name;
+    tool_call.arguments = arguments;
+    tool_call.id = id;
+
+    // LOG_DBG("Tool call arguments:\n\traw: %s\n\tresult: %s\n", arguments.c_str(), tool_call.arguments.c_str());
+    result_.tool_calls.emplace_back(tool_call);
+
+    return true;
+}
+bool common_chat_msg_parser::add_tool_call(const json & tool_call) {
+    std::string name = tool_call.contains("name") ? tool_call.at("name") : "";
+    std::string id = tool_call.contains("id") ? tool_call.at("id") : "";
+    std::string arguments = "";
+    if (tool_call.contains("arguments")) {
+        if (tool_call.at("arguments").is_object()) {
+            arguments = tool_call.at("arguments").dump();
+        } else {
+            arguments = tool_call.at("arguments");
+        }
+    }
+
+    return add_tool_call(name, id, arguments);
+}
+
+bool common_chat_msg_parser::add_tool_calls(const json & arr) {
+    for (const auto & item : arr) {
+        if (!add_tool_call(item)) {
+            return false;
+        }
+    }
+    return true;
+}
+void common_chat_msg_parser::finish() {
+    if (!is_partial_ && pos_ != input_.size()) {
+        throw std::runtime_error("Unexpected content at end of input");// + input_.substr(pos_));
+    }
+}
+
+bool common_chat_msg_parser::consume_spaces() {
+    const auto length = input_.size();
+    auto consumed = false;
+    while (pos_ < length && std::isspace(input_[pos_])) {
+        ++pos_;
+        consumed = true;
+    }
+    return consumed;
+}
+
+bool common_chat_msg_parser::try_consume_literal(const std::string & literal) {
+    auto pos = pos_;
+    for (auto i = 0u; i < literal.size(); ++i) {
+        if (pos >= input_.size()) {
+            return false;
+        }
+        if (input_[pos] != literal[i]) {
+            return false;
+        }
+        ++pos;
+    }
+    pos_ = pos;
+    return true;
+}
+
+std::optional<common_chat_msg_parser::find_regex_result>  common_chat_msg_parser::try_find_literal(const std::string & literal) {
+    auto idx = input_.find(literal, pos_);
+    if (idx != std::string::npos) {
+        find_regex_result res;
+        res.prelude = input_.substr(pos_, idx - pos_);
+        auto end = idx + literal.size();
+        res.groups.emplace_back(common_string_range{idx, end});
+        move_to(end);
+        return res;
+    }
+    if (is_partial_) {
+        idx = string_find_partial_stop(input_, literal);
+        if (idx != std::string::npos && idx >= pos_) {
+            find_regex_result res;
+            res.prelude = input_.substr(pos_, idx - pos_);
+            auto end = input_.size();
+            res.groups.emplace_back(common_string_range{idx, end});
+            move_to(end);
+            return res;
+        }
+    }
+    return std::nullopt;
+}
+
+void common_chat_msg_parser::consume_literal(const std::string & literal) {
+    if (!try_consume_literal(literal)) {
+        throw common_chat_msg_partial_exception(literal);
+    }
+}
+
+bool common_chat_msg_parser::try_parse_reasoning(const std::string & start_think, const std::string & end_think) {
+    auto handle_reasoning = [&](const std::string & reasoning, bool closed) {
+        auto stripped_reasoning = string_strip(reasoning);
+        if (stripped_reasoning.empty()) {
+            return;
+        }
+        if (syntax_.reasoning_in_content) {
+            add_content(syntax_.reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK ? "<think>" : start_think);
+            add_content(stripped_reasoning);
+            if (closed) {
+                add_content(syntax_.reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK ? "</think>" : end_think);
+            }
+        } else {
+            add_reasoning_content(stripped_reasoning);
+        }
+    };
+    if (syntax_.reasoning_format != COMMON_REASONING_FORMAT_NONE) {
+        if (syntax_.thinking_forced_open || try_consume_literal(start_think)) {
+            if (auto res = try_find_literal(end_think)) {
+                handle_reasoning(res->prelude, /* closed */ true);
+                consume_spaces();
+                return true;
+            }
+            auto rest = consume_rest();
+            if (!rest.empty()) {
+                handle_reasoning(rest, /* closed */ !is_partial());
+            }
+            // Allow unclosed thinking tags, for now (https://github.com/ggml-org/llama.cpp/issues/13812, https://github.com/ggml-org/llama.cpp/issues/13877)
+            // if (!syntax_.thinking_forced_open) {
+            //     throw common_chat_msg_partial_exception(end_think);
+            // }
+            return true;
+        }
+    }
+    return false;
+}
+
+std::string common_chat_msg_parser::consume_rest() {
+    auto rest = input_.substr(pos_);
+    pos_ = input_.size();
+    return rest;
+}
+
+// Tries to find the regex, consumes it (pos right after it) and gives the prelude (right before it) and the groups to the callback.
+std::optional<common_chat_msg_parser::find_regex_result> common_chat_msg_parser::try_find_regex(const common_regex & regex, size_t from, bool add_prelude_to_content) {
+    auto m = regex.search(input_, from == std::string::npos ? pos_ : from);
+    if (m.type == COMMON_REGEX_MATCH_TYPE_NONE) {
+        return std::nullopt;
+    }
+    auto prelude = input_.substr(pos_, m.groups[0].begin - pos_);
+    pos_ = m.groups[0].end;
+
+    if (add_prelude_to_content) {
+        add_content(prelude);
+    }
+    if (m.type == COMMON_REGEX_MATCH_TYPE_PARTIAL) {
+        if (is_partial()) {
+            throw common_chat_msg_partial_exception(regex.str());
+        }
+        return std::nullopt;
+    }
+    return find_regex_result{prelude, m.groups};
+}
+
+common_chat_msg_parser::find_regex_result common_chat_msg_parser::consume_regex(const common_regex & regex) {
+    if (auto result = try_consume_regex(regex)) {
+        return *result;
+    }
+    throw common_chat_msg_partial_exception(regex.str());
+}
+
+std::optional<common_chat_msg_parser::find_regex_result> common_chat_msg_parser::try_consume_regex(const common_regex & regex) {
+    auto m = regex.search(input_, pos_);
+    if (m.type == COMMON_REGEX_MATCH_TYPE_NONE) {
+        return std::nullopt;
+    }
+    if (m.type == COMMON_REGEX_MATCH_TYPE_PARTIAL) {
+        if (is_partial()) {
+            throw common_chat_msg_partial_exception(regex.str());
+        }
+        return std::nullopt;
+    }
+    if (m.groups[0].begin != pos_) {
+        // Didn't match at the current position.
+        return std::nullopt;
+    }
+    pos_ = m.groups[0].end;
+
+    return find_regex_result {
+        /* .prelude = */ "",
+        m.groups,
+    };
+}
+
+std::optional<common_json> common_chat_msg_parser::try_consume_json() {
+    auto it = input_.cbegin() + pos_;
+    const auto end = input_.cend();
+    common_json result;
+    if (!common_json_parse(it, end, healing_marker_, result)) {
+        return std::nullopt;
+    }
+    pos_ = std::distance(input_.cbegin(), it);
+    if (result.healing_marker.marker.empty()) {
+        // No healing marker, just return the parsed json
+        return result;
+    }
+    if (!is_partial()) {
+        throw common_chat_msg_partial_exception("JSON");
+    }
+    return result;
+}
+
+common_json common_chat_msg_parser::consume_json() {
+    if (auto result = try_consume_json()) {
+        return *result;
+    }
+    throw common_chat_msg_partial_exception("JSON");
+}
+
+common_chat_msg_parser::consume_json_result common_chat_msg_parser::consume_json_with_dumped_args(
+    const std::vector<std::vector<std::string>> & args_paths,
+    const std::vector<std::vector<std::string>> & content_paths
+) {
+    if (auto result = try_consume_json_with_dumped_args(args_paths, content_paths)) {
+        return *result;
+    }
+    throw common_chat_msg_partial_exception("JSON");
+}
+
+std::optional<common_chat_msg_parser::consume_json_result> common_chat_msg_parser::try_consume_json_with_dumped_args(
+    const std::vector<std::vector<std::string>> & args_paths,
+    const std::vector<std::vector<std::string>> & content_paths
+) {
+    auto partial = try_consume_json();
+    if (!partial) {
+        return std::nullopt;
+    }
+    auto is_arguments_path = [&](const std::vector<std::string> & path) {
+        return std::find(args_paths.begin(), args_paths.end(), path) != args_paths.end();
+    };
+    auto is_content_path = [&](const std::vector<std::string> & path) {
+        return std::find(content_paths.begin(), content_paths.end(), path) != content_paths.end();
+    };
+
+    if (partial->healing_marker.marker.empty()) {
+        if (args_paths.empty()) {
+            // No arguments to dump, and JSON was parsed fully.
+            return consume_json_result {
+                partial->json,
+                /* .is_partial = */ false,
+            };
+        }
+        if (is_arguments_path({})) {
+            // Entire JSON is the arguments and was parsed fully.
+            return consume_json_result {
+                partial->json.dump(),
+                /* .is_partial = */ false,
+            };
+        }
+    }
+
+    LOG_DBG("Parsed partial JSON: %s (json_healing_marker: %s)\n", partial->json.dump().c_str(), partial->healing_marker.json_dump_marker.c_str());
+
+    auto found_healing_marker = false;
+    std::vector<std::string> path;
+    std::function<json(const json &)> remove_unsupported_healings_and_dump_args = [&](const json & j) -> json {
+        if (is_arguments_path(path)) {
+            auto arguments = j.dump();
+            if (is_partial() && !partial->healing_marker.marker.empty()) {
+                auto idx = arguments.find(partial->healing_marker.json_dump_marker);
+                if (idx != std::string::npos) {
+                    arguments.resize(idx);
+                    found_healing_marker = true;
+                }
+                if (arguments == "\"") {
+                    // This happens because of completing `:"$magic` after `"arguments"`
+                    arguments = "";
+                }
+            }
+            return arguments;
+        }
+        if (is_content_path(path)) {
+            if (!j.is_string()) {
+                throw std::runtime_error("Content path must be a string");
+            }
+            std::string str = j;
+            auto idx = str.find(partial->healing_marker.marker); // not using json_dump_marker as we're inside a string
+            if (idx != std::string::npos) {
+                str.resize(idx);
+                found_healing_marker = true;
+            }
+            return str;
+        }
+        if (j.is_object()) {
+            auto obj = json::object();
+            for (const auto & p : j.items()) {
+                const auto & key = p.key();
+                const auto & value = p.value();
+                const std::string key_str = key; // NOLINT
+                auto idx = key_str.find(healing_marker_);
+                if (idx != std::string::npos) {
+                    found_healing_marker = true;
+                    break;
+                }
+                path.push_back(key_str);
+                if (value.is_string()) {
+                    const std::string value_str = value;
+                    if (value_str.find(healing_marker_) != std::string::npos) {
+                        found_healing_marker = true;
+                        if (is_content_path(path)) {
+                            if (partial->healing_marker.marker == partial->healing_marker.json_dump_marker) {
+                                // The healing occurred inside the string: good. Otherwise we just ditch the entire key/value pair.
+                                obj[key] = remove_unsupported_healings_and_dump_args(value);
+                            }
+                        }
+                        break;
+                    }
+                    obj[key] = value;
+                } else {
+                    obj[key] = remove_unsupported_healings_and_dump_args(value);
+                }
+                path.pop_back();
+            }
+            return obj;
+        }
+        if (j.is_array()) {
+            auto arr = json::array();
+            for (const auto & value : j) {
+                if (value.is_string()) {
+                    std::string str = value;
+                    auto idx = str.find(healing_marker_);
+                    if (idx != std::string::npos) {
+                        // Don't heal array values that aren't in the arguments.
+                        found_healing_marker = true;
+                        break;
+                    }
+                }
+                arr.push_back(remove_unsupported_healings_and_dump_args(value));
+            }
+            return arr;
+        }
+        return j;
+    };
+
+    auto cleaned = remove_unsupported_healings_and_dump_args(partial->json);
+    LOG_DBG("Cleaned up JSON %s to %s (json_healing_marker : '%s')\n", partial->json.dump().c_str(), cleaned.dump().c_str(), partial->healing_marker.json_dump_marker.c_str());
+    return consume_json_result {
+        cleaned,
+        /* .is_partial = */ found_healing_marker,
+    };
+}
+
+void common_chat_msg_parser::clear_tools() {
+    result_.tool_calls.clear();
+}