cui-llama.rn 1.5.0 → 1.6.0

package/cpp/rn-llama.cpp

@@ -1,873 +1,873 @@
-#include "rn-llama.h"
-
-namespace rnllama {
-
-const std::vector<lm_ggml_type> kv_cache_types = {
-    LM_GGML_TYPE_F32,
-    LM_GGML_TYPE_F16,
-    LM_GGML_TYPE_BF16,
-    LM_GGML_TYPE_Q8_0,
-    LM_GGML_TYPE_Q4_0,
-    LM_GGML_TYPE_Q4_1,
-    LM_GGML_TYPE_IQ4_NL,
-    LM_GGML_TYPE_Q5_0,
-    LM_GGML_TYPE_Q5_1,
-};
-
-lm_ggml_type kv_cache_type_from_str(const std::string & s) {
-    for (const auto & type : kv_cache_types) {
-        if (lm_ggml_type_name(type) == s) {
-            return type;
-        }
-    }
-    throw std::runtime_error("Unsupported cache type: " + s);
-}
-
-static void llama_batch_clear(llama_batch *batch) {
-    batch->n_tokens = 0;
-}
-
-static void llama_batch_add(llama_batch *batch, llama_token id, llama_pos pos, std::vector<llama_seq_id> seq_ids, bool logits) {
-    batch->token   [batch->n_tokens] = id;
-    batch->pos     [batch->n_tokens] = pos;
-    batch->n_seq_id[batch->n_tokens] = seq_ids.size();
-    for (size_t i = 0; i < seq_ids.size(); i++) {
-        batch->seq_id[batch->n_tokens][i] = seq_ids[i];
-    }
-    batch->logits  [batch->n_tokens] = logits ? 1 : 0;
-    batch->n_tokens += 1;
-}
-
-// NOTE: Edit from https://github.com/ggerganov/llama.cpp/blob/master/examples/server/server.cpp
-
-static void log(const char *level, const char *function, int line,
-                       const char *format, ...)
-{
-    va_list args;
-    #if defined(__ANDROID__)
-        char prefix[256];
-        snprintf(prefix, sizeof(prefix), "%s:%d %s", function, line, format);
-
-        va_start(args, format);
-        android_LogPriority priority;
-        if (strcmp(level, "ERROR") == 0) {
-            priority = ANDROID_LOG_ERROR;
-        } else if (strcmp(level, "WARNING") == 0) {
-            priority = ANDROID_LOG_WARN;
-        } else if (strcmp(level, "INFO") == 0) {
-            priority = ANDROID_LOG_INFO;
-        } else {
-            priority = ANDROID_LOG_DEBUG;
-        }
-        __android_log_vprint(priority, "RNLlama", prefix, args);
-        va_end(args);
-    #else
-        printf("[%s] %s:%d ", level, function, line);
-        va_start(args, format);
-        vprintf(format, args);
-        va_end(args);
-        printf("\n");
-    #endif
-}
-
-#if RNLLAMA_VERBOSE != 1
-#define LOG_VERBOSE(MSG, ...)
-#else
-#define LOG_VERBOSE(MSG, ...)                                       \
-    do                                                              \
-    {                                                               \
-        if (rnllama_verbose)                                        \
-        {                                                           \
-            log("VERBOSE", __func__, __LINE__, MSG, ##__VA_ARGS__); \
-        }                                                           \
-    } while (0)
-#endif
-
-#define LOG_ERROR(MSG, ...) log("ERROR", __func__, __LINE__, MSG, ##__VA_ARGS__)
-#define LOG_WARNING(MSG, ...) log("WARNING", __func__, __LINE__, MSG, ##__VA_ARGS__)
-#define LOG_INFO(MSG, ...) log("INFO", __func__, __LINE__, MSG, ##__VA_ARGS__)
-
-static size_t common_part(const std::vector<llama_token> &a, const std::vector<llama_token> &b)
-{
-    size_t i;
-    for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++)
-    {
-    }
-    return i;
-}
-
-static bool ends_with(const std::string &str, const std::string &suffix)
-{
-    return str.size() >= suffix.size() &&
-           0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix);
-}
-
-static size_t find_partial_stop_string(const std::string &stop,
-                                       const std::string &text)
-{
-    if (!text.empty() && !stop.empty())
-    {
-        const char text_last_char = text.back();
-        for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--)
-        {
-            if (stop[char_index] == text_last_char)
-            {
-                const std::string current_partial = stop.substr(0, char_index + 1);
-                if (ends_with(text, current_partial))
-                {
-                    return text.size() - char_index - 1;
-                }
-            }
-        }
-    }
-    return std::string::npos;
-}
-
-// format incomplete utf-8 multibyte character for output
-std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token)
-{
-    std::string out = token == -1 ? "" : common_token_to_piece(ctx, token);
-    // if the size is 1 and first bit is 1, meaning it's a partial character
-    //   (size > 1 meaning it's already a known token)
-    if (out.size() == 1 && (out[0] & 0x80) == 0x80)
-    {
-        std::stringstream ss;
-        ss << std::hex << (out[0] & 0xff);
-        std::string res(ss.str());
-        out = "byte: \\x" + res;
-    }
-    return out;
-}
-
-std::string tokens_to_str(llama_context *ctx, const std::vector<llama_token>::const_iterator begin, const std::vector<llama_token>::const_iterator end)
-{
-    std::string ret;
-    for (auto it = begin; it != end; ++it)
-    {
-        ret += common_token_to_piece(ctx, *it);
-    }
-    return ret;
-}
-
-llama_rn_context::~llama_rn_context() {
-    if (ctx_sampling != nullptr) {
-        common_sampler_free(ctx_sampling);
-    }
-}
-
-void llama_rn_context::rewind() {
-    is_interrupted = false;
-    params.antiprompt.clear();
-    params.sampling.grammar.clear();
-    num_prompt_tokens = 0;
-    num_tokens_predicted = 0;
-    generated_text = "";
-    generated_text.reserve(params.n_ctx);
-    generated_token_probs.clear();
-    truncated = false;
-    stopped_eos = false;
-    stopped_word = false;
-    stopped_limit = false;
-    stopping_word = "";
-    incomplete = false;
-    n_remain = 0;
-    n_past = 0;
-    params.sampling.n_prev = n_ctx;
-}
-
-bool llama_rn_context::initSampling() {
-    if (ctx_sampling != nullptr) {
-        common_sampler_free(ctx_sampling);
-    }
-    ctx_sampling = common_sampler_init(model, params.sampling);
-    return ctx_sampling != nullptr;
-}
-
-bool llama_rn_context::loadModel(common_params &params_)
-{
-    params = params_;
-    llama_init = common_init_from_params(params);
-    model = llama_init.model.get();
-    ctx = llama_init.context.get();
-    if (model == nullptr)
-    {
-        LOG_ERROR("unable to load model: %s", params_.model.path.c_str());
-        return false;
-    }
-    templates = common_chat_templates_init(model, params.chat_template);
-    n_ctx = llama_n_ctx(ctx);
-
-    // We can uncomment for debugging or after this fix: https://github.com/ggerganov/llama.cpp/pull/11101
-    // LOG_INFO("%s\n", common_params_get_system_info(params).c_str());
-
-    return true;
-}
-
-bool llama_rn_context::validateModelChatTemplate(bool use_jinja, const char *name) const {
-    const char * tmpl = llama_model_chat_template(model, name);
-    if (tmpl == nullptr) {
-      return false;
-    }
-    return common_chat_verify_template(tmpl, use_jinja);
-}
-
-common_chat_params llama_rn_context::getFormattedChatWithJinja(
-  const std::string &messages,
-  const std::string &chat_template,
-  const std::string &json_schema,
-  const std::string &tools,
-  const bool &parallel_tool_calls,
-  const std::string &tool_choice
-) const {
-    common_chat_templates_inputs inputs;
-    inputs.use_jinja = true;
-    inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(messages));
-    auto useTools = !tools.empty();
-    if (useTools) {
-        inputs.tools = common_chat_tools_parse_oaicompat(json::parse(tools));
-    }
-    inputs.parallel_tool_calls = parallel_tool_calls;
-    if (!tool_choice.empty()) {
-        inputs.tool_choice = common_chat_tool_choice_parse_oaicompat(tool_choice);
-    }
-    if (!json_schema.empty()) {
-        inputs.json_schema = json::parse(json_schema);
-    }
-    inputs.extract_reasoning = params.reasoning_format != COMMON_REASONING_FORMAT_NONE;
-
-    // If chat_template is provided, create new one and use it (probably slow)
-    if (!chat_template.empty()) {
-        auto tmps = common_chat_templates_init(model, chat_template);
-        return common_chat_templates_apply(tmps.get(), inputs);
-    } else {
-        return common_chat_templates_apply(templates.get(), inputs);
-    }
-}
-
-std::string llama_rn_context::getFormattedChat(
-  const std::string &messages,
-  const std::string &chat_template
-) const {
-    common_chat_templates_inputs inputs;
-    inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(messages));
-    inputs.use_jinja = false;
-
-    // If chat_template is provided, create new one and use it (probably slow)
-    if (!chat_template.empty()) {
-        auto tmps = common_chat_templates_init(model, chat_template);
-        return common_chat_templates_apply(tmps.get(), inputs).prompt;
-    } else {
-        return common_chat_templates_apply(templates.get(), inputs).prompt;
-    }
-}
-
-void llama_rn_context::truncatePrompt(std::vector<llama_token> &prompt_tokens) {
-    const int n_left = n_ctx - params.n_keep;
-    const int n_block_size = n_left / 2;
-    const int erased_blocks = (prompt_tokens.size() - params.n_keep - n_block_size) / n_block_size;
-
-    // Keep n_keep tokens at start of prompt (at most n_ctx - 4)
-    std::vector<llama_token> new_tokens(prompt_tokens.begin(), prompt_tokens.begin() + params.n_keep);
-
-    new_tokens.insert(new_tokens.end(), prompt_tokens.begin() + params.n_keep + erased_blocks * n_block_size, prompt_tokens.end());
-
-    LOG_VERBOSE("input truncated, n_ctx: %d, n_keep: %d, n_left: %d, new_tokens: %s, num_prompt_tokens: %d",
-        n_ctx,
-        params.n_keep,
-        n_left,
-        tokens_to_str(ctx, new_tokens.cbegin(), new_tokens.cend()).c_str(),
-        new_tokens.size()
-    );
-
-    truncated = true;
-    prompt_tokens = new_tokens;
-}
-
-void llama_rn_context::loadPrompt() {
-    std::vector<llama_token> prompt_tokens = ::common_tokenize(ctx, params.prompt, true, true);
-    num_prompt_tokens = prompt_tokens.size();
-
-    // LOG tokens
-    std::stringstream ss;
-    ss << "\n" << __func__ << ": prompt_tokens = ";
-    for (auto& token : prompt_tokens) {
-        ss << token << " ";
-    }
-    LOG_INFO("%s\n", ss.str().c_str());
-
-    if (params.n_keep < 0)
-    {
-        params.n_keep = (int)num_prompt_tokens;
-    }
-    params.n_keep = std::min(n_ctx - 4, params.n_keep);
-
-    // if input prompt is too big, truncate like normal
-    if (num_prompt_tokens >= (size_t) n_ctx)
-    {
-        truncatePrompt(prompt_tokens);
-        num_prompt_tokens = prompt_tokens.size();
-
-        LM_GGML_ASSERT(num_prompt_tokens < (size_t) n_ctx);
-    }
-
-    // do context shifitng
-    if(!params.embedding){
-        purge_missing_tokens(ctx, embd, prompt_tokens, params.n_predict, params.n_ctx);
-    }
-
-
-    // push the prompt into the sampling context (do not apply grammar)
-    for (auto & token : prompt_tokens)
-    {
-        common_sampler_accept(ctx_sampling, token, false);
-    }
-
-    // compare the evaluated prompt with the new prompt
-    n_past = common_part(embd, prompt_tokens);
-
-    embd = prompt_tokens;
-    if (n_past == num_prompt_tokens)
-    {
-        // we have to evaluate at least 1 token to generate logits.
-        n_past--;
-    }
-
-    // since #3228 we now have to manually manage the KV cache
-    llama_kv_self_seq_rm(ctx, 0, n_past, -1);
-
-    LOG_VERBOSE("prompt ingested, n_past: %d, cached: %s, to_eval: %s",
-        n_past,
-        tokens_to_str(ctx, embd.cbegin(), embd.cbegin() + n_past).c_str(),
-        tokens_to_str(ctx, embd.cbegin() + n_past, embd.cend()).c_str()
-    );
-
-    has_next_token = true;
-}
-
-void llama_rn_context::beginCompletion() {
-    // number of tokens to keep when resetting context
-    n_remain = params.n_predict;
-    llama_perf_context_reset(ctx);
-    is_predicting = true;
-}
-
-completion_token_output llama_rn_context::nextToken()
-{
-    completion_token_output result;
-    result.tok = -1;
-
-    if (embd.size() >= (size_t)params.n_ctx)
-    {
-        // Shift context
-
-        const int n_left    = n_past - params.n_keep - 1;
-        const int n_discard = n_left/2;
-
-        llama_kv_self_seq_rm (ctx, 0, params.n_keep + 1            , params.n_keep + n_discard + 1);
-        llama_kv_self_seq_add(ctx, 0, params.n_keep + 1 + n_discard, n_past, -n_discard);
-
-        for (size_t i = params.n_keep + 1 + n_discard; i < embd.size(); i++)
-        {
-            embd[i - n_discard] = embd[i];
-        }
-        embd.resize(embd.size() - n_discard);
-
-        n_past -= n_discard;
-
-        LOG_VERBOSE("input truncated, n_ctx: %d, n_keep: %d, n_left: %d, new_tokens: %s",
-            params.n_ctx,
-            params.n_keep,
-            n_left
-        );
-    }
-
-    bool tg = true;
-    while (n_past < embd.size())
-    {
-        int n_eval = (int)embd.size() - n_past;
-        tg = n_eval == 1;
-        if (n_eval > params.n_batch)
-        {
-            n_eval = params.n_batch;
-        }
-        if (llama_decode(ctx, llama_batch_get_one(&embd[n_past], n_eval)))
-        {
-            LOG_ERROR("failed to eval, n_eval: %d, n_past: %d, n_threads: %d, embd: %s",
-                n_eval,
-                n_past,
-                params.cpuparams.n_threads,
-                tokens_to_str(ctx, embd.cbegin() + n_past, embd.cend()).c_str()
-            );
-            has_next_token = false;
-            return result;
-        }
-        n_past += n_eval;
-
-        if(is_interrupted) {
-            LOG_INFO("Decoding Interrupted");
-            embd.resize(n_past);
-            has_next_token = false;
-            return result;
-        }
-    }
-
-    const llama_vocab* vocab = llama_model_get_vocab(model);
-
-    if (params.n_predict == 0)
-    {
-        has_next_token = false;
-        result.tok = llama_vocab_eos(vocab);
-        return result;
-    }
-
-    {
-        // out of user input, sample next token
-        std::vector<llama_token_data> candidates;
-        candidates.reserve(llama_vocab_n_tokens(vocab));
-
-        result.tok = common_sampler_sample(ctx_sampling, ctx, -1);
-
-        llama_token_data_array cur_p = *common_sampler_get_candidates(ctx_sampling);
-
-        const int32_t n_probs = params.sampling.n_probs;
-
-        // deprecated
-        /*if (params.sampling.temp <= 0 && n_probs > 0)
-        {
-            // For llama_sample_token_greedy we need to sort candidates
-            llama_sampler_init_softmax();
-
-        }*/
-
-
-        for (size_t i = 0; i < std::min(cur_p.size, (size_t)n_probs); ++i)
-        {
-            result.probs.push_back({cur_p.data[i].id, cur_p.data[i].p});
-        }
-
-        common_sampler_accept(ctx_sampling, result.tok, true);
-        if (tg) {
-            num_tokens_predicted++;
-        }
-    }
-
-    // add it to the context
-    embd.push_back(result.tok);
-    // decrement remaining sampling budget
-    --n_remain;
-
-    if (!embd.empty() && embd.back() == llama_vocab_eos(vocab))
-    {
-        // stopping_word = llama_token_to_piece(ctx, embd.back());
-        has_next_token = false;
-        stopped_eos = true;
-        LOG_VERBOSE("eos token found", "");
-        return result;
-    }
-
-    has_next_token = params.n_predict == -1 || n_remain != 0;
-    return result;
-}
-
-size_t llama_rn_context::findStoppingStrings(const std::string &text, const size_t last_token_size,
-                            const stop_type type)
-{
-    size_t stop_pos = std::string::npos;
-    for (const std::string &word : params.antiprompt)
-    {
-        size_t pos;
-        if (type == STOP_FULL)
-        {
-            const size_t tmp = word.size() + last_token_size;
-            const size_t from_pos = text.size() > tmp ? text.size() - tmp : 0;
-            pos = text.find(word, from_pos);
-        }
-        else
-        {
-            pos = find_partial_stop_string(word, text);
-        }
-        if (pos != std::string::npos &&
-            (stop_pos == std::string::npos || pos < stop_pos))
-        {
-            if (type == STOP_FULL)
-            {
-                stopping_word = word;
-                stopped_word = true;
-                has_next_token = false;
-            }
-            stop_pos = pos;
-        }
-    }
-    return stop_pos;
-}
-
-completion_token_output llama_rn_context::doCompletion()
-{
-    const completion_token_output token_with_probs = nextToken();
-
-    const std::string token_text = token_with_probs.tok == -1 ? "" : common_token_to_piece(ctx, token_with_probs.tok);
-    generated_text += token_text;
-
-    if (params.sampling.n_probs > 0)
-    {
-        generated_token_probs.push_back(token_with_probs);
-    }
-
-    // check if there is incomplete UTF-8 character at the end
-    for (unsigned i = 1; i < 5 && i <= generated_text.size(); ++i) {
-        unsigned char c = generated_text[generated_text.size() - i];
-        if ((c & 0xC0) == 0x80) {
-            // continuation byte: 10xxxxxx
-            continue;
-        }
-        if ((c & 0xE0) == 0xC0) {
-            // 2-byte character: 110xxxxx ...
-            incomplete = i < 2;
-        } else if ((c & 0xF0) == 0xE0) {
-            // 3-byte character: 1110xxxx ...
-            incomplete = i < 3;
-        } else if ((c & 0xF8) == 0xF0) {
-            // 4-byte character: 11110xxx ...
-            incomplete = i < 4;
-        }
-        // else 1-byte character or invalid byte
-        break;
-    }
-
-    if (incomplete && !has_next_token)
-    {
-        has_next_token = true;
-        n_remain++;
-    }
-
-    if (!has_next_token && n_remain == 0)
-    {
-        stopped_limit = true;
-    }
-
-    LOG_VERBOSE("next token, token: %s, token_text: %s, has_next_token: %d, n_remain: %d, num_tokens_predicted: %d, stopped_eos: %d, stopped_word: %d, stopped_limit: %d, stopping_word: %s",
-        common_token_to_piece(ctx, token_with_probs.tok),
-        tokens_to_output_formatted_string(ctx, token_with_probs.tok).c_str(),
-        has_next_token,
-        n_remain,
-        num_tokens_predicted,
-        stopped_eos,
-        stopped_word,
-        stopped_limit,
-        stopping_word.c_str()
-    );
-    return token_with_probs;
-}
-
-std::vector<float> llama_rn_context::getEmbedding(common_params &embd_params)
-{
-    static const int n_embd = llama_model_n_embd(llama_get_model(ctx));
-    if (!embd_params.embedding)
-    {
-        LOG_WARNING("embedding disabled, embedding: %s", embd_params.embedding);
-        return std::vector<float>(n_embd, 0.0f);
-    }
-    float *data;
-
-    const enum llama_pooling_type pooling_type = llama_pooling_type(ctx);
-    if (pooling_type == LLAMA_POOLING_TYPE_NONE) {
-        data = llama_get_embeddings(ctx);
-    } else {
-        data = llama_get_embeddings_seq(ctx, 0);
-    }
-
-    if (!data) {
-        return std::vector<float>(n_embd, 0.0f);
-    }
-    std::vector<float> embedding(data, data + n_embd), out(data, data + n_embd);
-    common_embd_normalize(embedding.data(), out.data(), n_embd, embd_params.embd_normalize);
-    return out;
-}
-
-std::string llama_rn_context::bench(int pp, int tg, int pl, int nr)
-{
-    if (is_predicting) {
-        LOG_ERROR("cannot benchmark while predicting", "");
-        return std::string("[]");
-    }
-
-    is_predicting = true;
-
-    double pp_avg = 0;
-    double tg_avg = 0;
-
-    double pp_std = 0;
-    double tg_std = 0;
-
-    // TODO: move batch into llama_rn_context (related https://github.com/mybigday/llama.rn/issues/30)
-    llama_batch batch = llama_batch_init(
-        std::min(pp, params.n_ubatch), // max n_tokens is limited by n_ubatch
-        0,                         // No embeddings
-        1                          // Single sequence
-    );
-
-    for (int i = 0; i < nr; i++)
-    {
-        llama_batch_clear(&batch);
-
-        const int n_tokens = pp;
-
-        for (int i = 0; i < n_tokens; i++)
-        {
-            llama_batch_add(&batch, 0, i, {0}, false);
-        }
-        batch.logits[batch.n_tokens - 1] = 1; // true
-
-        llama_kv_self_clear(ctx);
-
-        const int64_t t_pp_start = llama_time_us();
-        if (llama_decode(ctx, batch) != 0)
-        {
-            LOG_ERROR("llama_decode() failed during prompt", "");
-        }
-        const int64_t t_pp_end = llama_time_us();
-        llama_kv_self_clear(ctx);
-
-        if (is_interrupted) break;
-
-        const int64_t t_tg_start = llama_time_us();
-
-        for (int i = 0; i < tg; i++)
-        {
-            llama_batch_clear(&batch);
-
-            for (int j = 0; j < pl; j++)
-            {
-                llama_batch_add(&batch, 0, i, {j}, true);
-            }
-
-            if (llama_decode(ctx, batch) != 0)
-            {
-                LOG_ERROR("llama_decode() failed during text generation", "");
-            }
-            if (is_interrupted) break;
-        }
-
-        const int64_t t_tg_end = llama_time_us();
-
-        llama_kv_self_clear(ctx);
-
-        const double t_pp = (t_pp_end - t_pp_start) / 1000000.0;
-        const double t_tg = (t_tg_end - t_tg_start) / 1000000.0;
-
-        const double speed_pp = pp / t_pp;
-        const double speed_tg = (pl * tg) / t_tg;
-
-        pp_avg += speed_pp;
-        tg_avg += speed_tg;
-
-        pp_std += speed_pp * speed_pp;
-        tg_std += speed_tg * speed_tg;
-    }
-
-    pp_avg /= nr;
-    tg_avg /= nr;
-
-    if (nr > 1) {
-        pp_std = sqrt(pp_std / (nr - 1) - pp_avg * pp_avg * nr / (nr - 1));
-        tg_std = sqrt(tg_std / (nr - 1) - tg_avg * tg_avg * nr / (nr - 1));
-    } else {
-        pp_std = 0;
-        tg_std = 0;
-    }
-
-    if (is_interrupted) llama_kv_self_clear(ctx);
-    is_predicting = false;
-
-    char model_desc[128];
-    llama_model_desc(model, model_desc, sizeof(model_desc));
-    return std::string("[\"") + model_desc + std::string("\",") +
-        std::to_string(llama_model_size(model)) + std::string(",") +
-        std::to_string(llama_model_n_params(model)) + std::string(",") +
-        std::to_string(pp_avg) + std::string(",") +
-        std::to_string(pp_std) + std::string(",") +
-        std::to_string(tg_avg) + std::string(",") +
-        std::to_string(tg_std) +
-        std::string("]");
-}
-
-int llama_rn_context::applyLoraAdapters(std::vector<common_adapter_lora_info> lora) {
-    for (auto &la : lora) {
-        la.ptr = llama_adapter_lora_init(model, la.path.c_str());
-        if (la.ptr == nullptr) {
-            LOG_ERROR("failed to apply lora adapter '%s'\n", la.path.c_str());
-            return -1;
-        }
-    }
-    this->lora = lora;
-    common_set_adapter_lora(ctx, lora);
-    return 0;
-}
-
-void llama_rn_context::removeLoraAdapters() {
-    this->lora.clear();
-    common_set_adapter_lora(ctx, this->lora); // apply empty list
-}
-
-std::vector<common_adapter_lora_info> llama_rn_context::getLoadedLoraAdapters() {
-    return this->lora;
-}
-std::vector<int> llama_rn_context::longest_common_subseq(const std::vector<int> x, const std::vector<int> y){
-     int m = x.size(), n = y.size();
-
-     //int LCSuff[m+1][n+1];
-     std::vector<std::vector<int>> LCSuff(m+1, std::vector<int>(n+1));
-
-     for (int j = 0; j <= n; j++)
-         LCSuff[0][j] = 0;
-     for (int i = 0; i <= m; i++)
-         LCSuff[i][0] = 0;
-
-     for (int i = 1; i <= m; i++)
-     {
-         for (int j = 1; j <= n; j++)
-         {
-             if (x[i - 1] == y[j - 1])
-                 LCSuff[i][j] = LCSuff[i - 1][j - 1] + 1;
-             else
-                 LCSuff[i][j] = 0;
-         }
-     }
-
-     std::vector<int> longest;
-     for (int i = 1; i <= m; i++)
-     {
-         for (int j = 1; j <= n; j++)
-         {
-             if (LCSuff[i][j] > longest.size())
-             {
-                 auto off1 = ((i - LCSuff[i][j] + 1) - 1);
-                 auto off2 = off1 + LCSuff[i][j];
-                 longest.clear();
-                //  std::vector<int>().swap(longest);
-                 longest = std::vector<int>(x.begin() + off1, x.begin() + off2);
-                // x.substr((i - LCSuff[i][j] + 1) - 1, LCSuff[i][j]);
-             }
-         }
-     }
-     return longest;
- }
-
-bool llama_rn_context::arr_start_with(const std::vector<int> targetArray, const std::vector<int> searchSeq)
- {
-     int ss = searchSeq.size();
-     if(targetArray.size()<ss)
-     {
-         return false;
-     }
-     for(int i=0;i<ss;++i)
-     {
-         if(targetArray[i]!=searchSeq[i])
-         {
-             return false;
-         }
-     }
-     return true;
- }
-
-int llama_rn_context::arr_find_index_of(const std::vector<int> targetArray, const std::vector<int> searchSeq)
- {
-     int ss = searchSeq.size();
-     int tas = targetArray.size();
-     if(tas<ss)
-     {
-         return -1;
-     }
-     for(int i=0;i<tas;++i)
-     {
-         int srch = 0;
-         bool fail = false;
-         for(int srch=0;srch<ss;++srch)
-         {
-             if ((i + srch) >= tas || targetArray[i + srch] != searchSeq[srch])
-             {
-                 fail = true;
-                 break;
-             }
-         }
-         if(!fail)
-         {
-             return i;
-         }
-     }
-     return -1;
- }
-
-void llama_rn_context::purge_missing_tokens(llama_context * ctx, std::vector<int> &current_context_tokens, std::vector<int> &new_context_tokens, const int genamt, const int nctx)
-{
-    //scan from start old and new ctx, until first mismatch found, save as p0
-    //check remaining old and new ctx for longest common subseq, which needs to be at 256 tokens
-    //test: longest common subseq (LCQ) MUST start within 0 tokens from end of memory, otherwise purge fails
-    //if passed, save beginning of LCQ from old ctx as p1
-    //remove all tokens from old ctx between p0 and p1, updating both arrays and kv, then continue as normal
-
-    const int short_fall_threshold = 200 + (nctx/30); //dont trigger shifting if the distance between trimstart and currhead < this
-    const int stack_allowance = 60 + (nctx/50); //in case the end text is slightly modified, be forgiving
-
-    int trimstart = 0;
-    int new_tokens_len = new_context_tokens.size();
-    bool purge_needed = true;
-
-    for (int i = 0; i < current_context_tokens.size(); ++i)
-    {
-        if (current_context_tokens[i] == new_context_tokens[i])
-        {
-            trimstart += 1;
-        }
-        else
-        {
-            break;
-        }
-        if ((i + 2) >= new_tokens_len)
-        {
-            purge_needed = false;
-            break; //no surgery required
-        }
-    }
-
-    
-
-    if(!purge_needed || new_tokens_len < 6 || current_context_tokens.size() < 6 || new_tokens_len - trimstart < short_fall_threshold)
-    {
-        LOG_INFO("Fall Threshold: %d out of %d\n", new_tokens_len - trimstart, short_fall_threshold);
-        return; //no purge is needed
-    }
-
-    //at least this many tokens need to match, otherwise don't bother trimming
-    const int lc_tok_threshold = std::max(std::min((new_tokens_len - trimstart) - (genamt+stack_allowance), (int)(nctx*0.45)), short_fall_threshold - stack_allowance);
-
-    auto curr_ctx_without_memory = std::vector<int>(current_context_tokens.begin() + trimstart, current_context_tokens.end());
-    auto new_ctx_without_memory = std::vector<int>(new_context_tokens.begin() + trimstart, new_context_tokens.end());
-
-    auto shared = longest_common_subseq(curr_ctx_without_memory, new_ctx_without_memory);
-
-    if (shared.size() > lc_tok_threshold && arr_start_with(new_ctx_without_memory, shared)) // enough tokens in common
-    {
-        int found = arr_find_index_of(current_context_tokens,shared);
-        if(found>=0 && found > trimstart)
-        {
-
-            //extract the unwanted tokens out from context and KV
-            int diff = found - trimstart;
-            llama_kv_self_seq_rm(ctx, 0, trimstart, trimstart + diff);
-            llama_kv_self_seq_add(ctx, 0, trimstart + diff, -1, -diff);
-
-            for (size_t i = trimstart + diff; i < current_context_tokens.size() - 1; i++)
-            {
-                current_context_tokens[i - diff] = current_context_tokens[i];
-            }
-
-            LOG_INFO("\n[Context Shifting: Erased %d tokens at position %d]", diff, trimstart + 1);
-
-            current_context_tokens.resize(current_context_tokens.size() - diff);
-        }
-    }
-
-}
-
-}
+#include "rn-llama.h"
+
+namespace rnllama {
+
+const std::vector<lm_ggml_type> kv_cache_types = {
+    LM_GGML_TYPE_F32,
+    LM_GGML_TYPE_F16,
+    LM_GGML_TYPE_BF16,
+    LM_GGML_TYPE_Q8_0,
+    LM_GGML_TYPE_Q4_0,
+    LM_GGML_TYPE_Q4_1,
+    LM_GGML_TYPE_IQ4_NL,
+    LM_GGML_TYPE_Q5_0,
+    LM_GGML_TYPE_Q5_1,
+};
+
+lm_ggml_type kv_cache_type_from_str(const std::string & s) {
+    for (const auto & type : kv_cache_types) {
+        if (lm_ggml_type_name(type) == s) {
+            return type;
+        }
+    }
+    throw std::runtime_error("Unsupported cache type: " + s);
+}
+
+static void llama_batch_clear(llama_batch *batch) {
+    batch->n_tokens = 0;
+}
+
+static void llama_batch_add(llama_batch *batch, llama_token id, llama_pos pos, std::vector<llama_seq_id> seq_ids, bool logits) {
+    batch->token   [batch->n_tokens] = id;
+    batch->pos     [batch->n_tokens] = pos;
+    batch->n_seq_id[batch->n_tokens] = seq_ids.size();
+    for (size_t i = 0; i < seq_ids.size(); i++) {
+        batch->seq_id[batch->n_tokens][i] = seq_ids[i];
+    }
+    batch->logits  [batch->n_tokens] = logits ? 1 : 0;
+    batch->n_tokens += 1;
+}
+
+// NOTE: Edit from https://github.com/ggerganov/llama.cpp/blob/master/examples/server/server.cpp
+
+static void log(const char *level, const char *function, int line,
+                       const char *format, ...)
+{
+    va_list args;
+    #if defined(__ANDROID__)
+        char prefix[256];
+        snprintf(prefix, sizeof(prefix), "%s:%d %s", function, line, format);
+
+        va_start(args, format);
+        android_LogPriority priority;
+        if (strcmp(level, "ERROR") == 0) {
+            priority = ANDROID_LOG_ERROR;
+        } else if (strcmp(level, "WARNING") == 0) {
+            priority = ANDROID_LOG_WARN;
+        } else if (strcmp(level, "INFO") == 0) {
+            priority = ANDROID_LOG_INFO;
+        } else {
+            priority = ANDROID_LOG_DEBUG;
+        }
+        __android_log_vprint(priority, "RNLlama", prefix, args);
+        va_end(args);
+    #else
+        printf("[%s] %s:%d ", level, function, line);
+        va_start(args, format);
+        vprintf(format, args);
+        va_end(args);
+        printf("\n");
+    #endif
+}
+
+#if RNLLAMA_VERBOSE != 1
+#define LOG_VERBOSE(MSG, ...)
+#else
+#define LOG_VERBOSE(MSG, ...)                                       \
+    do                                                              \
+    {                                                               \
+        if (rnllama_verbose)                                        \
+        {                                                           \
+            log("VERBOSE", __func__, __LINE__, MSG, ##__VA_ARGS__); \
+        }                                                           \
+    } while (0)
+#endif
+
+#define LOG_ERROR(MSG, ...) log("ERROR", __func__, __LINE__, MSG, ##__VA_ARGS__)
+#define LOG_WARNING(MSG, ...) log("WARNING", __func__, __LINE__, MSG, ##__VA_ARGS__)
+#define LOG_INFO(MSG, ...) log("INFO", __func__, __LINE__, MSG, ##__VA_ARGS__)
+
+static size_t common_part(const std::vector<llama_token> &a, const std::vector<llama_token> &b)
+{
+    size_t i;
+    for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++)
+    {
+    }
+    return i;
+}
+
+static bool ends_with(const std::string &str, const std::string &suffix)
+{
+    return str.size() >= suffix.size() &&
+           0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix);
+}
+
+static size_t find_partial_stop_string(const std::string &stop,
+                                       const std::string &text)
+{
+    if (!text.empty() && !stop.empty())
+    {
+        const char text_last_char = text.back();
+        for (int64_t char_index = stop.size() - 1; char_index >= 0; char_index--)
+        {
+            if (stop[char_index] == text_last_char)
+            {
+                const std::string current_partial = stop.substr(0, char_index + 1);
+                if (ends_with(text, current_partial))
+                {
+                    return text.size() - char_index - 1;
+                }
+            }
+        }
+    }
+    return std::string::npos;
+}
+
+// format incomplete utf-8 multibyte character for output
+std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token)
+{
+    std::string out = token == -1 ? "" : common_token_to_piece(ctx, token);
+    // if the size is 1 and first bit is 1, meaning it's a partial character
+    //   (size > 1 meaning it's already a known token)
+    if (out.size() == 1 && (out[0] & 0x80) == 0x80)
+    {
+        std::stringstream ss;
+        ss << std::hex << (out[0] & 0xff);
+        std::string res(ss.str());
+        out = "byte: \\x" + res;
+    }
+    return out;
+}
+
+std::string tokens_to_str(llama_context *ctx, const std::vector<llama_token>::const_iterator begin, const std::vector<llama_token>::const_iterator end)
+{
+    std::string ret;
+    for (auto it = begin; it != end; ++it)
+    {
+        ret += common_token_to_piece(ctx, *it);
+    }
+    return ret;
+}
+
+llama_rn_context::~llama_rn_context() {
+    if (ctx_sampling != nullptr) {
+        common_sampler_free(ctx_sampling);
+    }
+}
+
+void llama_rn_context::rewind() {
+    is_interrupted = false;
+    params.antiprompt.clear();
+    params.sampling.grammar.clear();
+    num_prompt_tokens = 0;
+    num_tokens_predicted = 0;
+    generated_text = "";
+    generated_text.reserve(params.n_ctx);
+    generated_token_probs.clear();
+    truncated = false;
+    stopped_eos = false;
+    stopped_word = false;
+    stopped_limit = false;
+    stopping_word = "";
+    incomplete = false;
+    n_remain = 0;
+    n_past = 0;
+    params.sampling.n_prev = n_ctx;
+}
+
+bool llama_rn_context::initSampling() {
+    if (ctx_sampling != nullptr) {
+        common_sampler_free(ctx_sampling);
+    }
+    ctx_sampling = common_sampler_init(model, params.sampling);
+    return ctx_sampling != nullptr;
+}
+
+bool llama_rn_context::loadModel(common_params &params_)
+{
+    params = params_;
+    llama_init = common_init_from_params(params);
+    model = llama_init.model.get();
+    ctx = llama_init.context.get();
+    if (model == nullptr)
+    {
+        LOG_ERROR("unable to load model: %s", params_.model.path.c_str());
+        return false;
+    }
+    templates = common_chat_templates_init(model, params.chat_template);
+    n_ctx = llama_n_ctx(ctx);
+
+    // We can uncomment for debugging or after this fix: https://github.com/ggerganov/llama.cpp/pull/11101
+    // LOG_INFO("%s\n", common_params_get_system_info(params).c_str());
+
+    return true;
+}
+
+bool llama_rn_context::validateModelChatTemplate(bool use_jinja, const char *name) const {
+    const char * tmpl = llama_model_chat_template(model, name);
+    if (tmpl == nullptr) {
+      return false;
+    }
+    return common_chat_verify_template(tmpl, use_jinja);
+}
+
+common_chat_params llama_rn_context::getFormattedChatWithJinja(
+  const std::string &messages,
+  const std::string &chat_template,
+  const std::string &json_schema,
+  const std::string &tools,
+  const bool &parallel_tool_calls,
+  const std::string &tool_choice
+) const {
+    common_chat_templates_inputs inputs;
+    inputs.use_jinja = true;
+    inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(messages));
+    auto useTools = !tools.empty();
+    if (useTools) {
+        inputs.tools = common_chat_tools_parse_oaicompat(json::parse(tools));
+    }
+    inputs.parallel_tool_calls = parallel_tool_calls;
+    if (!tool_choice.empty()) {
+        inputs.tool_choice = common_chat_tool_choice_parse_oaicompat(tool_choice);
+    }
+    if (!json_schema.empty()) {
+        inputs.json_schema = json::parse(json_schema);
+    }
+    inputs.extract_reasoning = params.reasoning_format != COMMON_REASONING_FORMAT_NONE;
+
+    // If chat_template is provided, create new one and use it (probably slow)
+    if (!chat_template.empty()) {
+        auto tmps = common_chat_templates_init(model, chat_template);
+        return common_chat_templates_apply(tmps.get(), inputs);
+    } else {
+        return common_chat_templates_apply(templates.get(), inputs);
+    }
+}
+
+std::string llama_rn_context::getFormattedChat(
+  const std::string &messages,
+  const std::string &chat_template
+) const {
+    common_chat_templates_inputs inputs;
+    inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(messages));
+    inputs.use_jinja = false;
+
+    // If chat_template is provided, create new one and use it (probably slow)
+    if (!chat_template.empty()) {
+        auto tmps = common_chat_templates_init(model, chat_template);
+        return common_chat_templates_apply(tmps.get(), inputs).prompt;
+    } else {
+        return common_chat_templates_apply(templates.get(), inputs).prompt;
+    }
+}
+
+void llama_rn_context::truncatePrompt(std::vector<llama_token> &prompt_tokens) {
+    const int n_left = n_ctx - params.n_keep;
+    const int n_block_size = n_left / 2;
+    const int erased_blocks = (prompt_tokens.size() - params.n_keep - n_block_size) / n_block_size;
+
+    // Keep n_keep tokens at start of prompt (at most n_ctx - 4)
+    std::vector<llama_token> new_tokens(prompt_tokens.begin(), prompt_tokens.begin() + params.n_keep);
+
+    new_tokens.insert(new_tokens.end(), prompt_tokens.begin() + params.n_keep + erased_blocks * n_block_size, prompt_tokens.end());
+
+    LOG_VERBOSE("input truncated, n_ctx: %d, n_keep: %d, n_left: %d, new_tokens: %s, num_prompt_tokens: %d",
+        n_ctx,
+        params.n_keep,
+        n_left,
+        tokens_to_str(ctx, new_tokens.cbegin(), new_tokens.cend()).c_str(),
+        new_tokens.size()
+    );
+
+    truncated = true;
+    prompt_tokens = new_tokens;
+}
+
+void llama_rn_context::loadPrompt() {
+    std::vector<llama_token> prompt_tokens = ::common_tokenize(ctx, params.prompt, true, true);
+    num_prompt_tokens = prompt_tokens.size();
+
+    // LOG tokens
+    std::stringstream ss;
+    ss << "\n" << __func__ << ": prompt_tokens = ";
+    for (auto& token : prompt_tokens) {
+        ss << token << " ";
+    }
+    LOG_INFO("%s\n", ss.str().c_str());
+
+    if (params.n_keep < 0)
+    {
+        params.n_keep = (int)num_prompt_tokens;
+    }
+    params.n_keep = std::min(n_ctx - 4, params.n_keep);
+
+    // if input prompt is too big, truncate like normal
+    if (num_prompt_tokens >= (size_t) n_ctx)
+    {
+        truncatePrompt(prompt_tokens);
+        num_prompt_tokens = prompt_tokens.size();
+
+        LM_GGML_ASSERT(num_prompt_tokens < (size_t) n_ctx);
+    }
+
+    // do context shifitng
+    if(!params.embedding){
+        purge_missing_tokens(ctx, embd, prompt_tokens, params.n_predict, params.n_ctx);
+    }
+
+
+    // push the prompt into the sampling context (do not apply grammar)
+    for (auto & token : prompt_tokens)
+    {
+        common_sampler_accept(ctx_sampling, token, false);
+    }
+
+    // compare the evaluated prompt with the new prompt
+    n_past = common_part(embd, prompt_tokens);
+
+    embd = prompt_tokens;
+    if (n_past == num_prompt_tokens)
+    {
+        // we have to evaluate at least 1 token to generate logits.
+        n_past--;
+    }
+
+    // since #3228 we now have to manually manage the KV cache
+    llama_kv_self_seq_rm(ctx, 0, n_past, -1);
+
+    LOG_VERBOSE("prompt ingested, n_past: %d, cached: %s, to_eval: %s",
+        n_past,
+        tokens_to_str(ctx, embd.cbegin(), embd.cbegin() + n_past).c_str(),
+        tokens_to_str(ctx, embd.cbegin() + n_past, embd.cend()).c_str()
+    );
+
+    has_next_token = true;
+}
+
+void llama_rn_context::beginCompletion() {
+    // number of tokens to keep when resetting context
+    n_remain = params.n_predict;
+    llama_perf_context_reset(ctx);
+    is_predicting = true;
+}
+
+completion_token_output llama_rn_context::nextToken()
+{
+    completion_token_output result;
+    result.tok = -1;
+
+    if (embd.size() >= (size_t)params.n_ctx)
+    {
+        // Shift context
+
+        const int n_left    = n_past - params.n_keep - 1;
+        const int n_discard = n_left/2;
+
+        llama_kv_self_seq_rm (ctx, 0, params.n_keep + 1            , params.n_keep + n_discard + 1);
+        llama_kv_self_seq_add(ctx, 0, params.n_keep + 1 + n_discard, n_past, -n_discard);
+
+        for (size_t i = params.n_keep + 1 + n_discard; i < embd.size(); i++)
+        {
+            embd[i - n_discard] = embd[i];
+        }
+        embd.resize(embd.size() - n_discard);
+
+        n_past -= n_discard;
+
+        LOG_VERBOSE("input truncated, n_ctx: %d, n_keep: %d, n_left: %d, new_tokens: %s",
+            params.n_ctx,
+            params.n_keep,
+            n_left
+        );
+    }
+
+    bool tg = true;
+    while (n_past < embd.size())
+    {
+        int n_eval = (int)embd.size() - n_past;
+        tg = n_eval == 1;
+        if (n_eval > params.n_batch)
+        {
+            n_eval = params.n_batch;
+        }
+        if (llama_decode(ctx, llama_batch_get_one(&embd[n_past], n_eval)))
+        {
+            LOG_ERROR("failed to eval, n_eval: %d, n_past: %d, n_threads: %d, embd: %s",
+                n_eval,
+                n_past,
+                params.cpuparams.n_threads,
+                tokens_to_str(ctx, embd.cbegin() + n_past, embd.cend()).c_str()
+            );
+            has_next_token = false;
+            return result;
+        }
+        n_past += n_eval;
+
+        if(is_interrupted) {
+            LOG_INFO("Decoding Interrupted");
+            embd.resize(n_past);
+            has_next_token = false;
+            return result;
+        }
+    }
+
+    const llama_vocab* vocab = llama_model_get_vocab(model);
+
+    if (params.n_predict == 0)
+    {
+        has_next_token = false;
+        result.tok = llama_vocab_eos(vocab);
+        return result;
+    }
+
+    {
+        // out of user input, sample next token
+        std::vector<llama_token_data> candidates;
+        candidates.reserve(llama_vocab_n_tokens(vocab));
+
+        result.tok = common_sampler_sample(ctx_sampling, ctx, -1);
+
+        llama_token_data_array cur_p = *common_sampler_get_candidates(ctx_sampling);
+
+        const int32_t n_probs = params.sampling.n_probs;
+
+        // deprecated
+        /*if (params.sampling.temp <= 0 && n_probs > 0)
+        {
+            // For llama_sample_token_greedy we need to sort candidates
+            llama_sampler_init_softmax();
+
+        }*/
+
+
+        for (size_t i = 0; i < std::min(cur_p.size, (size_t)n_probs); ++i)
+        {
+            result.probs.push_back({cur_p.data[i].id, cur_p.data[i].p});
+        }
+
+        common_sampler_accept(ctx_sampling, result.tok, true);
+        if (tg) {
+            num_tokens_predicted++;
+        }
+    }
+
+    // add it to the context
+    embd.push_back(result.tok);
+    // decrement remaining sampling budget
+    --n_remain;
+
+    if (!embd.empty() && embd.back() == llama_vocab_eos(vocab))
+    {
+        // stopping_word = llama_token_to_piece(ctx, embd.back());
+        has_next_token = false;
+        stopped_eos = true;
+        LOG_VERBOSE("eos token found", "");
+        return result;
+    }
+
+    has_next_token = params.n_predict == -1 || n_remain != 0;
+    return result;
+}
+
+size_t llama_rn_context::findStoppingStrings(const std::string &text, const size_t last_token_size,
+                            const stop_type type)
+{
+    size_t stop_pos = std::string::npos;
+    for (const std::string &word : params.antiprompt)
+    {
+        size_t pos;
+        if (type == STOP_FULL)
+        {
+            const size_t tmp = word.size() + last_token_size;
+            const size_t from_pos = text.size() > tmp ? text.size() - tmp : 0;
+            pos = text.find(word, from_pos);
+        }
+        else
+        {
+            pos = find_partial_stop_string(word, text);
+        }
+        if (pos != std::string::npos &&
+            (stop_pos == std::string::npos || pos < stop_pos))
+        {
+            if (type == STOP_FULL)
+            {
+                stopping_word = word;
+                stopped_word = true;
+                has_next_token = false;
+            }
+            stop_pos = pos;
+        }
+    }
+    return stop_pos;
+}
+
+completion_token_output llama_rn_context::doCompletion()
+{
+    const completion_token_output token_with_probs = nextToken();
+
+    const std::string token_text = token_with_probs.tok == -1 ? "" : common_token_to_piece(ctx, token_with_probs.tok);
+    generated_text += token_text;
+
+    if (params.sampling.n_probs > 0)
+    {
+        generated_token_probs.push_back(token_with_probs);
+    }
+
+    // check if there is incomplete UTF-8 character at the end
+    for (unsigned i = 1; i < 5 && i <= generated_text.size(); ++i) {
+        unsigned char c = generated_text[generated_text.size() - i];
+        if ((c & 0xC0) == 0x80) {
+            // continuation byte: 10xxxxxx
+            continue;
+        }
+        if ((c & 0xE0) == 0xC0) {
+            // 2-byte character: 110xxxxx ...
+            incomplete = i < 2;
+        } else if ((c & 0xF0) == 0xE0) {
+            // 3-byte character: 1110xxxx ...
+            incomplete = i < 3;
+        } else if ((c & 0xF8) == 0xF0) {
+            // 4-byte character: 11110xxx ...
+            incomplete = i < 4;
+        }
+        // else 1-byte character or invalid byte
+        break;
+    }
+
+    if (incomplete && !has_next_token)
+    {
+        has_next_token = true;
+        n_remain++;
+    }
+
+    if (!has_next_token && n_remain == 0)
+    {
+        stopped_limit = true;
+    }
+
+    LOG_VERBOSE("next token, token: %s, token_text: %s, has_next_token: %d, n_remain: %d, num_tokens_predicted: %d, stopped_eos: %d, stopped_word: %d, stopped_limit: %d, stopping_word: %s",
+        common_token_to_piece(ctx, token_with_probs.tok),
+        tokens_to_output_formatted_string(ctx, token_with_probs.tok).c_str(),
+        has_next_token,
+        n_remain,
+        num_tokens_predicted,
+        stopped_eos,
+        stopped_word,
+        stopped_limit,
+        stopping_word.c_str()
+    );
+    return token_with_probs;
+}
+
+std::vector<float> llama_rn_context::getEmbedding(common_params &embd_params)
+{
+    static const int n_embd = llama_model_n_embd(llama_get_model(ctx));
+    if (!embd_params.embedding)
+    {
+        LOG_WARNING("embedding disabled, embedding: %s", embd_params.embedding);
+        return std::vector<float>(n_embd, 0.0f);
+    }
+    float *data;
+
+    const enum llama_pooling_type pooling_type = llama_pooling_type(ctx);
+    if (pooling_type == LLAMA_POOLING_TYPE_NONE) {
+        data = llama_get_embeddings(ctx);
+    } else {
+        data = llama_get_embeddings_seq(ctx, 0);
+    }
+
+    if (!data) {
+        return std::vector<float>(n_embd, 0.0f);
+    }
+    std::vector<float> embedding(data, data + n_embd), out(data, data + n_embd);
+    common_embd_normalize(embedding.data(), out.data(), n_embd, embd_params.embd_normalize);
+    return out;
+}
+
+std::string llama_rn_context::bench(int pp, int tg, int pl, int nr)
+{
+    if (is_predicting) {
+        LOG_ERROR("cannot benchmark while predicting", "");
+        return std::string("[]");
+    }
+
+    is_predicting = true;
+
+    double pp_avg = 0;
+    double tg_avg = 0;
+
+    double pp_std = 0;
+    double tg_std = 0;
+
+    // TODO: move batch into llama_rn_context (related https://github.com/mybigday/llama.rn/issues/30)
+    llama_batch batch = llama_batch_init(
+        std::min(pp, params.n_ubatch), // max n_tokens is limited by n_ubatch
+        0,                         // No embeddings
+        1                          // Single sequence
+    );
+
+    for (int i = 0; i < nr; i++)
+    {
+        llama_batch_clear(&batch);
+
+        const int n_tokens = pp;
+
+        for (int i = 0; i < n_tokens; i++)
+        {
+            llama_batch_add(&batch, 0, i, {0}, false);
+        }
+        batch.logits[batch.n_tokens - 1] = 1; // true
+
+        llama_kv_self_clear(ctx);
+
+        const int64_t t_pp_start = llama_time_us();
+        if (llama_decode(ctx, batch) != 0)
+        {
+            LOG_ERROR("llama_decode() failed during prompt", "");
+        }
+        const int64_t t_pp_end = llama_time_us();
+        llama_kv_self_clear(ctx);
+
+        if (is_interrupted) break;
+
+        const int64_t t_tg_start = llama_time_us();
+
+        for (int i = 0; i < tg; i++)
+        {
+            llama_batch_clear(&batch);
+
+            for (int j = 0; j < pl; j++)
+            {
+                llama_batch_add(&batch, 0, i, {j}, true);
+            }
+
+            if (llama_decode(ctx, batch) != 0)
+            {
+                LOG_ERROR("llama_decode() failed during text generation", "");
+            }
+            if (is_interrupted) break;
+        }
+
+        const int64_t t_tg_end = llama_time_us();
+
+        llama_kv_self_clear(ctx);
+
+        const double t_pp = (t_pp_end - t_pp_start) / 1000000.0;
+        const double t_tg = (t_tg_end - t_tg_start) / 1000000.0;
+
+        const double speed_pp = pp / t_pp;
+        const double speed_tg = (pl * tg) / t_tg;
+
+        pp_avg += speed_pp;
+        tg_avg += speed_tg;
+
+        pp_std += speed_pp * speed_pp;
+        tg_std += speed_tg * speed_tg;
+    }
+
+    pp_avg /= nr;
+    tg_avg /= nr;
+
+    if (nr > 1) {
+        pp_std = sqrt(pp_std / (nr - 1) - pp_avg * pp_avg * nr / (nr - 1));
+        tg_std = sqrt(tg_std / (nr - 1) - tg_avg * tg_avg * nr / (nr - 1));
+    } else {
+        pp_std = 0;
+        tg_std = 0;
+    }
+
+    if (is_interrupted) llama_kv_self_clear(ctx);
+    is_predicting = false;
+
+    char model_desc[128];
+    llama_model_desc(model, model_desc, sizeof(model_desc));
+    return std::string("[\"") + model_desc + std::string("\",") +
+        std::to_string(llama_model_size(model)) + std::string(",") +
+        std::to_string(llama_model_n_params(model)) + std::string(",") +
+        std::to_string(pp_avg) + std::string(",") +
+        std::to_string(pp_std) + std::string(",") +
+        std::to_string(tg_avg) + std::string(",") +
+        std::to_string(tg_std) +
+        std::string("]");
+}
+
+int llama_rn_context::applyLoraAdapters(std::vector<common_adapter_lora_info> lora) {
+    for (auto &la : lora) {
+        la.ptr = llama_adapter_lora_init(model, la.path.c_str());
+        if (la.ptr == nullptr) {
+            LOG_ERROR("failed to apply lora adapter '%s'\n", la.path.c_str());
+            return -1;
+        }
+    }
+    this->lora = lora;
+    common_set_adapter_lora(ctx, lora);
+    return 0;
+}
+
+void llama_rn_context::removeLoraAdapters() {
+    this->lora.clear();
+    common_set_adapter_lora(ctx, this->lora); // apply empty list
+}
+
+std::vector<common_adapter_lora_info> llama_rn_context::getLoadedLoraAdapters() {
+    return this->lora;
+}
+std::vector<int> llama_rn_context::longest_common_subseq(const std::vector<int> x, const std::vector<int> y){
+     int m = x.size(), n = y.size();
+
+     //int LCSuff[m+1][n+1];
+     std::vector<std::vector<int>> LCSuff(m+1, std::vector<int>(n+1));
+
+     for (int j = 0; j <= n; j++)
+         LCSuff[0][j] = 0;
+     for (int i = 0; i <= m; i++)
+         LCSuff[i][0] = 0;
+
+     for (int i = 1; i <= m; i++)
+     {
+         for (int j = 1; j <= n; j++)
+         {
+             if (x[i - 1] == y[j - 1])
+                 LCSuff[i][j] = LCSuff[i - 1][j - 1] + 1;
+             else
+                 LCSuff[i][j] = 0;
+         }
+     }
+
+     std::vector<int> longest;
+     for (int i = 1; i <= m; i++)
+     {
+         for (int j = 1; j <= n; j++)
+         {
+             if (LCSuff[i][j] > longest.size())
+             {
+                 auto off1 = ((i - LCSuff[i][j] + 1) - 1);
+                 auto off2 = off1 + LCSuff[i][j];
+                 longest.clear();
+                //  std::vector<int>().swap(longest);
+                 longest = std::vector<int>(x.begin() + off1, x.begin() + off2);
+                // x.substr((i - LCSuff[i][j] + 1) - 1, LCSuff[i][j]);
+             }
+         }
+     }
+     return longest;
+ }
+
+bool llama_rn_context::arr_start_with(const std::vector<int> targetArray, const std::vector<int> searchSeq)
+ {
+     int ss = searchSeq.size();
+     if(targetArray.size()<ss)
+     {
+         return false;
+     }
+     for(int i=0;i<ss;++i)
+     {
+         if(targetArray[i]!=searchSeq[i])
+         {
+             return false;
+         }
+     }
+     return true;
+ }
+
+int llama_rn_context::arr_find_index_of(const std::vector<int> targetArray, const std::vector<int> searchSeq)
+ {
+     int ss = searchSeq.size();
+     int tas = targetArray.size();
+     if(tas<ss)
+     {
+         return -1;
+     }
+     for(int i=0;i<tas;++i)
+     {
+         int srch = 0;
+         bool fail = false;
+         for(int srch=0;srch<ss;++srch)
+         {
+             if ((i + srch) >= tas || targetArray[i + srch] != searchSeq[srch])
+             {
+                 fail = true;
+                 break;
+             }
+         }
+         if(!fail)
+         {
+             return i;
+         }
+     }
+     return -1;
+ }
+
+void llama_rn_context::purge_missing_tokens(llama_context * ctx, std::vector<int> &current_context_tokens, std::vector<int> &new_context_tokens, const int genamt, const int nctx)
+{
+    //scan from start old and new ctx, until first mismatch found, save as p0
+    //check remaining old and new ctx for longest common subseq, which needs to be at 256 tokens
+    //test: longest common subseq (LCQ) MUST start within 0 tokens from end of memory, otherwise purge fails
+    //if passed, save beginning of LCQ from old ctx as p1
+    //remove all tokens from old ctx between p0 and p1, updating both arrays and kv, then continue as normal
+
+    const int short_fall_threshold = 200 + (nctx/30); //dont trigger shifting if the distance between trimstart and currhead < this
+    const int stack_allowance = 60 + (nctx/50); //in case the end text is slightly modified, be forgiving
+
+    int trimstart = 0;
+    int new_tokens_len = new_context_tokens.size();
+    bool purge_needed = true;
+
+    for (int i = 0; i < current_context_tokens.size(); ++i)
+    {
+        if (current_context_tokens[i] == new_context_tokens[i])
+        {
+            trimstart += 1;
+        }
+        else
+        {
+            break;
+        }
+        if ((i + 2) >= new_tokens_len)
+        {
+            purge_needed = false;
+            break; //no surgery required
+        }
+    }
+
+    
+
+    if(!purge_needed || new_tokens_len < 6 || current_context_tokens.size() < 6 || new_tokens_len - trimstart < short_fall_threshold)
+    {
+        LOG_INFO("Fall Threshold: %d out of %d\n", new_tokens_len - trimstart, short_fall_threshold);
+        return; //no purge is needed
+    }
+
+    //at least this many tokens need to match, otherwise don't bother trimming
+    const int lc_tok_threshold = std::max(std::min((new_tokens_len - trimstart) - (genamt+stack_allowance), (int)(nctx*0.45)), short_fall_threshold - stack_allowance);
+
+    auto curr_ctx_without_memory = std::vector<int>(current_context_tokens.begin() + trimstart, current_context_tokens.end());
+    auto new_ctx_without_memory = std::vector<int>(new_context_tokens.begin() + trimstart, new_context_tokens.end());
+
+    auto shared = longest_common_subseq(curr_ctx_without_memory, new_ctx_without_memory);
+
+    if (shared.size() > lc_tok_threshold && arr_start_with(new_ctx_without_memory, shared)) // enough tokens in common
+    {
+        int found = arr_find_index_of(current_context_tokens,shared);
+        if(found>=0 && found > trimstart)
+        {
+
+            //extract the unwanted tokens out from context and KV
+            int diff = found - trimstart;
+            llama_kv_self_seq_rm(ctx, 0, trimstart, trimstart + diff);
+            llama_kv_self_seq_add(ctx, 0, trimstart + diff, -1, -diff);
+
+            for (size_t i = trimstart + diff; i < current_context_tokens.size() - 1; i++)
+            {
+                current_context_tokens[i - diff] = current_context_tokens[i];
+            }
+
+            LOG_INFO("\n[Context Shifting: Erased %d tokens at position %d]", diff, trimstart + 1);
+
+            current_context_tokens.resize(current_context_tokens.size() - diff);
+        }
+    }
+
+}
+
+}