cui-llama.rn 1.4.0 → 1.4.2

package/cpp/rn-llama.h

@@ -0,0 +1,123 @@
+#ifndef RNLLAMA_H
+#define RNLLAMA_H
+
+#include <sstream>
+#include <iostream>
+#include "common.h"
+#include "ggml.h"
+#include "gguf.h"
+#include "llama.h"
+#include "llama-impl.h"
+#include "sampling.h"
+#if defined(__ANDROID__)
+#include <android/log.h>
+#endif
+
+namespace rnllama {
+
+std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token);
+
+std::string tokens_to_str(llama_context *ctx, const std::vector<llama_token>::const_iterator begin, const std::vector<llama_token>::const_iterator end);
+
+lm_ggml_type kv_cache_type_from_str(const std::string & s);
+
+enum stop_type
+{
+    STOP_FULL,
+    STOP_PARTIAL,
+};
+
+// completion token output with probabilities
+struct completion_token_output
+{
+    struct token_prob
+    {
+        llama_token tok;
+        float prob;
+    };
+
+    std::vector<token_prob> probs;
+    llama_token tok;
+};
+
+// Main context class
+struct llama_rn_context {
+    bool is_predicting = false;
+    bool is_interrupted = false;
+    bool has_next_token = false;
+    std::string generated_text;
+    std::vector<completion_token_output> generated_token_probs;
+
+    size_t num_prompt_tokens = 0;
+    size_t num_tokens_predicted = 0;
+    size_t n_past = 0;
+    size_t n_remain = 0;
+
+    std::vector<llama_token> embd;
+    common_params params;
+    common_init_result llama_init;
+
+    llama_model *model = nullptr;
+    float loading_progress = 0;
+    bool is_load_interrupted = false;
+
+    llama_context *ctx = nullptr;
+    common_sampler *ctx_sampling = nullptr;
+
+    int n_ctx;
+
+    bool truncated = false;
+    bool stopped_eos = false;
+    bool stopped_word = false;
+    bool stopped_limit = false;
+    std::string stopping_word;
+    bool incomplete = false;
+
+    std::vector<common_adapter_lora_info> lora;
+
+    ~llama_rn_context();
+
+    void rewind();
+    bool initSampling();
+    bool loadModel(common_params &params_);
+    bool validateModelChatTemplate() const;
+    void truncatePrompt(std::vector<llama_token> &prompt_tokens);
+    void loadPrompt();
+    void beginCompletion();
+    completion_token_output nextToken();
+    size_t findStoppingStrings(const std::string &text, const size_t last_token_size, const stop_type type);
+    completion_token_output doCompletion();
+    std::vector<float> getEmbedding(common_params &embd_params);
+    std::string bench(int pp, int tg, int pl, int nr);
+    int applyLoraAdapters(std::vector<common_adapter_lora_info> lora);
+    void removeLoraAdapters();
+    std::vector<common_adapter_lora_info> getLoadedLoraAdapters();
+    std::vector<int> longest_common_subseq(const std::vector<int> x, const std::vector<int> y);
+    bool arr_start_with(const std::vector<int> targetArray, const std::vector<int> searchSeq);
+    int arr_find_index_of(const std::vector<int> targetArray, const std::vector<int> searchSeq);
+    void purge_missing_tokens(llama_context * ctx, std::vector<int> &current_context_tokens, std::vector<int> &new_context_tokens, const int genamt, const int nctx);
+};\
+
+// Logging macros
+extern bool rnllama_verbose;
+
+#if RNLLAMA_VERBOSE != 1
+#define LOG_VERBOSE(MSG, ...)
+#else
+#define LOG_VERBOSE(MSG, ...)                                       \
+    do                                                              \
+    {                                                               \
+        if (rnllama_verbose)                                        \
+        {                                                           \
+            log("VERBOSE", __func__, __LINE__, MSG, ##__VA_ARGS__); \
+        }                                                           \
+    } while (0)
+#endif
+
+#define LOG_ERROR(MSG, ...) log("ERROR", __func__, __LINE__, MSG, ##__VA_ARGS__)
+#define LOG_WARNING(MSG, ...) log("WARNING", __func__, __LINE__, MSG, ##__VA_ARGS__)
+#define LOG_INFO(MSG, ...) log("INFO", __func__, __LINE__, MSG, ##__VA_ARGS__)
+
+} // namespace rnllama
+
+#endif /* RNLLAMA_H */

package/cpp/rn-llama.hpp

@@ -219,7 +219,7 @@ struct llama_rn_context
     std::string stopping_word;
     bool incomplete = false;
 
-    std::vector<common_lora_adapter_info> lora;
+    std::vector<common_adapter_lora_info> lora;
 
     ~llama_rn_context()
     {
@@ -279,7 +279,7 @@ struct llama_rn_context
 
     bool validateModelChatTemplate() const {
         llama_chat_message chat[] = {{"user", "test"}};
-        int32_t chat_res = llama_chat_apply_template(model, nullptr, chat, 1, true, nullptr, 0);
+        int32_t chat_res = llama_chat_apply_template(llama_model_chat_template(model), chat, 1, true, nullptr, 0);
         return chat_res > 0;
     }
 
@@ -307,7 +307,7 @@ struct llama_rn_context
 
     void loadPrompt()
     {
-        std::vector<llama_token> prompt_tokens = ::common_tokenize(model, params.prompt, true, true);
+        std::vector<llama_token> prompt_tokens = ::common_tokenize(llama_model_get_vocab(model), params.prompt, true, true);
         num_prompt_tokens = prompt_tokens.size();
 
         // LOG tokens
@@ -439,14 +439,14 @@ struct llama_rn_context
         if (params.n_predict == 0)
         {
             has_next_token = false;
-            result.tok = llama_token_eos(model);
+            result.tok = llama_vocab_eos(llama_model_get_vocab(model));
             return result;
         }
 
         {
             // out of user input, sample next token
             std::vector<llama_token_data> candidates;
-            candidates.reserve(llama_n_vocab(model));
+            candidates.reserve(llama_vocab_n_tokens(llama_model_get_vocab(model)));
 
             result.tok = common_sampler_sample(ctx_sampling, ctx, -1);
 
@@ -479,7 +479,7 @@ struct llama_rn_context
         // decrement remaining sampling budget
         --n_remain;
 
-        if (!embd.empty() && embd.back() == llama_token_eos(model))
+        if (!embd.empty() && embd.back() == llama_vocab_eos(llama_model_get_vocab(model)))
         {
             // stopping_word = llama_token_to_piece(ctx, embd.back());
             has_next_token = false;
@@ -584,7 +584,7 @@ struct llama_rn_context
 
     std::vector<float> getEmbedding(common_params &embd_params)
     {
-        static const int n_embd = llama_n_embd(llama_get_model(ctx));
+        static const int n_embd = llama_model_n_embd(llama_get_model(ctx));
         if (!embd_params.embedding)
         {
             LOG_WARNING("embedding disabled, embedding: %s", embd_params.embedding);
@@ -716,25 +716,31 @@ struct llama_rn_context
             std::string("]");
     }
 
-    int applyLoraAdapters(std::vector<common_lora_adapter_info> lora) {
+    int applyLoraAdapters(std::vector<common_adapter_lora_info> lora) {
         for (auto &la : lora) {
-            la.ptr = llama_lora_adapter_init(model, la.path.c_str());
+            la.ptr = llama_adapter_lora_init(model, la.path.c_str());
             if (la.ptr == nullptr) {
                 LOG_ERROR("failed to apply lora adapter '%s'\n", la.path.c_str());
                 return -1;
             }
         }
         this->lora = lora;
-        common_lora_adapters_apply(ctx, lora);
+        for (auto &la : lora) {
+            llama_set_adapter_lora(ctx, la.ptr, 1);
+        }
+
         return 0;
     }
 
     void removeLoraAdapters() {
+        for (auto &la : this->lora) {
+            llama_adapter_lora_free(la.ptr);
+        }
         this->lora.clear();
-        common_lora_adapters_apply(ctx, this->lora); // apply empty list
+        llama_clear_adapter_lora(ctx);
     }
 
-    std::vector<common_lora_adapter_info> getLoadedLoraAdapters() {
+    std::vector<common_adapter_lora_info> getLoadedLoraAdapters() {
         return this->lora;
     }
 // Context Shifting from KoboldCpp <https://github.com/LostRuins/koboldcpp>