cui-llama.rn 1.5.0 → 1.6.0

package/cpp/rn-llama.h

@@ -1,138 +1,138 @@
-#ifndef RNLLAMA_H
-#define RNLLAMA_H
-
-#include <sstream>
-#include <iostream>
-#include "chat.h"
-#include "common.h"
-#include "ggml.h"
-#include "gguf.h"
-#include "llama.h"
-#include "llama-impl.h"
-#include "sampling.h"
-#if defined(__ANDROID__)
-#include <android/log.h>
-#endif
-
-namespace rnllama {
-
-
-std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token);
-
-std::string tokens_to_str(llama_context *ctx, const std::vector<llama_token>::const_iterator begin, const std::vector<llama_token>::const_iterator end);
-
-lm_ggml_type kv_cache_type_from_str(const std::string & s);
-
-enum stop_type
-{
-    STOP_FULL,
-    STOP_PARTIAL,
-};
-
-// completion token output with probabilities
-struct completion_token_output
-{
-    struct token_prob
-    {
-        llama_token tok;
-        float prob;
-    };
-
-    std::vector<token_prob> probs;
-    llama_token tok;
-};
-
-// Main context class
-struct llama_rn_context {
-    bool is_predicting = false;
-    bool is_interrupted = false;
-    bool has_next_token = false;
-    std::string generated_text;
-    std::vector<completion_token_output> generated_token_probs;
-
-    size_t num_prompt_tokens = 0;
-    size_t num_tokens_predicted = 0;
-    size_t n_past = 0;
-    size_t n_remain = 0;
-
-    std::vector<llama_token> embd;
-    common_params params;
-    common_init_result llama_init;
-
-    llama_model *model = nullptr;
-    float loading_progress = 0;
-    bool is_load_interrupted = false;
-
-    llama_context *ctx = nullptr;
-    common_sampler *ctx_sampling = nullptr;
-    common_chat_templates_ptr templates;
-
-    int n_ctx;
-
-    bool truncated = false;
-    bool stopped_eos = false;
-    bool stopped_word = false;
-    bool stopped_limit = false;
-    std::string stopping_word;
-    bool incomplete = false;
-
-    std::vector<common_adapter_lora_info> lora;
-
-    ~llama_rn_context();
-
-    void rewind();
-    bool initSampling();
-    bool loadModel(common_params &params_);
-    bool validateModelChatTemplate(bool use_jinja, const char *name) const;
-    common_chat_params getFormattedChatWithJinja(
-      const std::string &messages,
-      const std::string &chat_template,
-      const std::string &json_schema,
-      const std::string &tools,
-      const bool &parallel_tool_calls,
-      const std::string &tool_choice
-    ) const;
-    std::string getFormattedChat(
-      const std::string &messages,
-      const std::string &chat_template
-    ) const;
-    void truncatePrompt(std::vector<llama_token> &prompt_tokens);
-    void loadPrompt();
-    void beginCompletion();
-    completion_token_output nextToken();
-    size_t findStoppingStrings(const std::string &text, const size_t last_token_size, const stop_type type);
-    completion_token_output doCompletion();
-    std::vector<float> getEmbedding(common_params &embd_params);
-    std::string bench(int pp, int tg, int pl, int nr);
-    int applyLoraAdapters(std::vector<common_adapter_lora_info> lora);
-    void removeLoraAdapters();
-    std::vector<common_adapter_lora_info> getLoadedLoraAdapters();
-    std::vector<int> longest_common_subseq(const std::vector<int> x, const std::vector<int> y);
-    bool arr_start_with(const std::vector<int> targetArray, const std::vector<int> searchSeq);
-    int arr_find_index_of(const std::vector<int> targetArray, const std::vector<int> searchSeq);
-    void purge_missing_tokens(llama_context * ctx, std::vector<int> &current_context_tokens, std::vector<int> &new_context_tokens, const int genamt, const int nctx);
-};\
-
-// Logging macros
-extern bool rnllama_verbose;
-
-#if RNLLAMA_VERBOSE != 1
-#define LOG_VERBOSE(MSG, ...)
-#else
-#define LOG_VERBOSE(MSG, ...)                                       \
-    do                                                              \
-    {                                                               \
-        if (rnllama_verbose)                                        \
-        {                                                           \
-            log("VERBOSE", __func__, __LINE__, MSG, ##__VA_ARGS__); \
-        }                                                           \
-    } while (0)
-#endif
-
-#define LOG_ERROR(MSG, ...) log("ERROR", __func__, __LINE__, MSG, ##__VA_ARGS__)
-#define LOG_WARNING(MSG, ...) log("WARNING", __func__, __LINE__, MSG, ##__VA_ARGS__)
-#define LOG_INFO(MSG, ...) log("INFO", __func__, __LINE__, MSG, ##__VA_ARGS__)
-
-} // namespace rnllama
-
-#endif /* RNLLAMA_H */
+#ifndef RNLLAMA_H
+#define RNLLAMA_H
+
+#include <sstream>
+#include <iostream>
+#include "chat.h"
+#include "common.h"
+#include "ggml.h"
+#include "gguf.h"
+#include "llama.h"
+#include "llama-impl.h"
+#include "sampling.h"
+#if defined(__ANDROID__)
+#include <android/log.h>
+#endif
+
+namespace rnllama {
+
+
+std::string tokens_to_output_formatted_string(const llama_context *ctx, const llama_token token);
+
+std::string tokens_to_str(llama_context *ctx, const std::vector<llama_token>::const_iterator begin, const std::vector<llama_token>::const_iterator end);
+
+lm_ggml_type kv_cache_type_from_str(const std::string & s);
+
+enum stop_type
+{
+    STOP_FULL,
+    STOP_PARTIAL,
+};
+
+// completion token output with probabilities
+struct completion_token_output
+{
+    struct token_prob
+    {
+        llama_token tok;
+        float prob;
+    };
+
+    std::vector<token_prob> probs;
+    llama_token tok;
+};
+
+// Main context class
+struct llama_rn_context {
+    bool is_predicting = false;
+    bool is_interrupted = false;
+    bool has_next_token = false;
+    std::string generated_text;
+    std::vector<completion_token_output> generated_token_probs;
+
+    size_t num_prompt_tokens = 0;
+    size_t num_tokens_predicted = 0;
+    size_t n_past = 0;
+    size_t n_remain = 0;
+
+    std::vector<llama_token> embd;
+    common_params params;
+    common_init_result llama_init;
+
+    llama_model *model = nullptr;
+    float loading_progress = 0;
+    bool is_load_interrupted = false;
+
+    llama_context *ctx = nullptr;
+    common_sampler *ctx_sampling = nullptr;
+    common_chat_templates_ptr templates;
+
+    int n_ctx;
+
+    bool truncated = false;
+    bool stopped_eos = false;
+    bool stopped_word = false;
+    bool stopped_limit = false;
+    std::string stopping_word;
+    bool incomplete = false;
+
+    std::vector<common_adapter_lora_info> lora;
+
+    ~llama_rn_context();
+
+    void rewind();
+    bool initSampling();
+    bool loadModel(common_params &params_);
+    bool validateModelChatTemplate(bool use_jinja, const char *name) const;
+    common_chat_params getFormattedChatWithJinja(
+      const std::string &messages,
+      const std::string &chat_template,
+      const std::string &json_schema,
+      const std::string &tools,
+      const bool &parallel_tool_calls,
+      const std::string &tool_choice
+    ) const;
+    std::string getFormattedChat(
+      const std::string &messages,
+      const std::string &chat_template
+    ) const;
+    void truncatePrompt(std::vector<llama_token> &prompt_tokens);
+    void loadPrompt();
+    void beginCompletion();
+    completion_token_output nextToken();
+    size_t findStoppingStrings(const std::string &text, const size_t last_token_size, const stop_type type);
+    completion_token_output doCompletion();
+    std::vector<float> getEmbedding(common_params &embd_params);
+    std::string bench(int pp, int tg, int pl, int nr);
+    int applyLoraAdapters(std::vector<common_adapter_lora_info> lora);
+    void removeLoraAdapters();
+    std::vector<common_adapter_lora_info> getLoadedLoraAdapters();
+    std::vector<int> longest_common_subseq(const std::vector<int> x, const std::vector<int> y);
+    bool arr_start_with(const std::vector<int> targetArray, const std::vector<int> searchSeq);
+    int arr_find_index_of(const std::vector<int> targetArray, const std::vector<int> searchSeq);
+    void purge_missing_tokens(llama_context * ctx, std::vector<int> &current_context_tokens, std::vector<int> &new_context_tokens, const int genamt, const int nctx);
+};\
+
+// Logging macros
+extern bool rnllama_verbose;
+
+#if RNLLAMA_VERBOSE != 1
+#define LOG_VERBOSE(MSG, ...)
+#else
+#define LOG_VERBOSE(MSG, ...)                                       \
+    do                                                              \
+    {                                                               \
+        if (rnllama_verbose)                                        \
+        {                                                           \
+            log("VERBOSE", __func__, __LINE__, MSG, ##__VA_ARGS__); \
+        }                                                           \
+    } while (0)
+#endif
+
+#define LOG_ERROR(MSG, ...) log("ERROR", __func__, __LINE__, MSG, ##__VA_ARGS__)
+#define LOG_WARNING(MSG, ...) log("WARNING", __func__, __LINE__, MSG, ##__VA_ARGS__)
+#define LOG_INFO(MSG, ...) log("INFO", __func__, __LINE__, MSG, ##__VA_ARGS__)
+
+} // namespace rnllama
+
+#endif /* RNLLAMA_H */

package/cpp/sampling.h

@@ -1,107 +1,107 @@
-#pragma once
-
-#include "llama.h"
-
-#include "common.h"
-
-#include <string>
-#include <vector>
-
-// common_sampler extends llama_sampler with additional functionality:
-//
-//  - grammar support
-//  - custom sampler logic based on the parameters
-//  - history of the last accepted tokens
-//  - performance metrics
-//
-// This goal is to have a common implementation of the sampling logic shared across the examples.
-// For example, depending on the temperature, the sampling chain can be very simple (greedy) or more
-// complex (top-k, top-p, etc).
-//
-// Another example is related to the grammar. In general, the grammar constraints applied on the full
-// vocabulary can be very taxing. To improve performance, the grammar can be applied only to the sampled
-// token in order to verify if it fits the grammar. And only if the token doesn't fit the grammar, the
-// grammar constraints are applied to the full vocabulary and the token is resampled.
-//
-// The common_sampler also maintains a container with the last accepted tokens. In the future, this can
-// be moved into the core llama library.
-//
-// For convenience, the common_sampler also maintains a container with the current candidate tokens.
-// This can be used to access the probabilities of the rest of the non-sampled tokens.
-//
-// TODO: measure grammar performance
-//
-
-struct common_sampler;
-
-// llama_sampler API overloads
-
-struct common_sampler * common_sampler_init(const struct llama_model * model, const struct common_params_sampling & params);
-
-void common_sampler_free(struct common_sampler * gsmpl);
-
-// if accept_grammar is true, the token is accepted both by the sampling chain and the grammar
-void                    common_sampler_accept(struct common_sampler * gsmpl, llama_token token, bool accept_grammar);
-void                    common_sampler_reset (struct common_sampler * gsmpl);
-struct common_sampler * common_sampler_clone (struct common_sampler * gsmpl);
-
-// arguments can be nullptr to skip printing
-void common_perf_print(const struct llama_context * ctx, const struct common_sampler * gsmpl);
-
-// extended sampling implementation:
-//
-// - set logits
-// - apply the configured sampler chain
-// - check if the token fits the grammar (if any)
-// - if not: resample by first applying the grammar constraints and then sampling again (slower path)
-//
-// if grammar_first is true, the grammar is applied before the samplers (slower)
-// useful in cases where all the resulting candidates (not just the sampled one) must fit the grammar
-//
-llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_context * ctx, int idx, bool grammar_first = false);
-
-// generalized version of common_sampler_sample
-//
-// will cross-reference the sampled tokens with a batch of draft tokens and accept those that match
-// if the sampler disagrees at some point, we stop and return the accepted tokens up to now
-//
-//      common_sampler_sample_n(gsmpl, ctx, { idx }, {});
-//
-// is equivalent to
-//
-//      common_sampler_sample(gsmpl, ctx, idx);
-//      common_sampler_accept(gsmpl, token, true);
-//
-// requires: idxs.size() == draft.size() + 1
-//
-// returns at least 1 token, up to idxs.size()
-//
-std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sampler * gsmpl, struct llama_context * ctx, const std::vector<int> & idxs, const llama_tokens & draft, bool grammar_first = false);
-
-// assume idxs == [ 0, 1, 2, ..., draft.size() ]
-std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sampler * gsmpl, struct llama_context * ctx, const llama_tokens & draft, bool grammar_first = false);
-
-uint32_t common_sampler_get_seed(const struct common_sampler * gsmpl);
-
-// helpers
-
-// access the internal list of current candidate tokens
-llama_token_data_array * common_sampler_get_candidates(struct common_sampler * gsmpl);
-
-// get the last accepted token
-llama_token common_sampler_last(const struct common_sampler * gsmpl);
-
-// print the sampler chain into a string
-std::string common_sampler_print(const struct common_sampler * gsmpl);
-
-// get a string representation of the last accepted tokens
-std::string common_sampler_prev_str(common_sampler * gsmpl, llama_context * ctx, int n);
-
-char        common_sampler_type_to_chr(enum common_sampler_type cnstr);
-std::string common_sampler_type_to_str(enum common_sampler_type cnstr);
-
-std::vector<enum common_sampler_type> common_sampler_types_from_names(const std::vector<std::string> & names, bool allow_alt_names);
-std::vector<enum common_sampler_type> common_sampler_types_from_chars(const std::string & chars);
-
-llama_sampler * llama_sampler_init_llg(const llama_vocab * vocab,
-                const char * grammar_kind, const char * grammar_data);
+#pragma once
+
+#include "llama.h"
+
+#include "common.h"
+
+#include <string>
+#include <vector>
+
+// common_sampler extends llama_sampler with additional functionality:
+//
+//  - grammar support
+//  - custom sampler logic based on the parameters
+//  - history of the last accepted tokens
+//  - performance metrics
+//
+// This goal is to have a common implementation of the sampling logic shared across the examples.
+// For example, depending on the temperature, the sampling chain can be very simple (greedy) or more
+// complex (top-k, top-p, etc).
+//
+// Another example is related to the grammar. In general, the grammar constraints applied on the full
+// vocabulary can be very taxing. To improve performance, the grammar can be applied only to the sampled
+// token in order to verify if it fits the grammar. And only if the token doesn't fit the grammar, the
+// grammar constraints are applied to the full vocabulary and the token is resampled.
+//
+// The common_sampler also maintains a container with the last accepted tokens. In the future, this can
+// be moved into the core llama library.
+//
+// For convenience, the common_sampler also maintains a container with the current candidate tokens.
+// This can be used to access the probabilities of the rest of the non-sampled tokens.
+//
+// TODO: measure grammar performance
+//
+
+struct common_sampler;
+
+// llama_sampler API overloads
+
+struct common_sampler * common_sampler_init(const struct llama_model * model, const struct common_params_sampling & params);
+
+void common_sampler_free(struct common_sampler * gsmpl);
+
+// if accept_grammar is true, the token is accepted both by the sampling chain and the grammar
+void                    common_sampler_accept(struct common_sampler * gsmpl, llama_token token, bool accept_grammar);
+void                    common_sampler_reset (struct common_sampler * gsmpl);
+struct common_sampler * common_sampler_clone (struct common_sampler * gsmpl);
+
+// arguments can be nullptr to skip printing
+void common_perf_print(const struct llama_context * ctx, const struct common_sampler * gsmpl);
+
+// extended sampling implementation:
+//
+// - set logits
+// - apply the configured sampler chain
+// - check if the token fits the grammar (if any)
+// - if not: resample by first applying the grammar constraints and then sampling again (slower path)
+//
+// if grammar_first is true, the grammar is applied before the samplers (slower)
+// useful in cases where all the resulting candidates (not just the sampled one) must fit the grammar
+//
+llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_context * ctx, int idx, bool grammar_first = false);
+
+// generalized version of common_sampler_sample
+//
+// will cross-reference the sampled tokens with a batch of draft tokens and accept those that match
+// if the sampler disagrees at some point, we stop and return the accepted tokens up to now
+//
+//      common_sampler_sample_n(gsmpl, ctx, { idx }, {});
+//
+// is equivalent to
+//
+//      common_sampler_sample(gsmpl, ctx, idx);
+//      common_sampler_accept(gsmpl, token, true);
+//
+// requires: idxs.size() == draft.size() + 1
+//
+// returns at least 1 token, up to idxs.size()
+//
+std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sampler * gsmpl, struct llama_context * ctx, const std::vector<int> & idxs, const llama_tokens & draft, bool grammar_first = false);
+
+// assume idxs == [ 0, 1, 2, ..., draft.size() ]
+std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sampler * gsmpl, struct llama_context * ctx, const llama_tokens & draft, bool grammar_first = false);
+
+uint32_t common_sampler_get_seed(const struct common_sampler * gsmpl);
+
+// helpers
+
+// access the internal list of current candidate tokens
+llama_token_data_array * common_sampler_get_candidates(struct common_sampler * gsmpl);
+
+// get the last accepted token
+llama_token common_sampler_last(const struct common_sampler * gsmpl);
+
+// print the sampler chain into a string
+std::string common_sampler_print(const struct common_sampler * gsmpl);
+
+// get a string representation of the last accepted tokens
+std::string common_sampler_prev_str(common_sampler * gsmpl, llama_context * ctx, int n);
+
+char        common_sampler_type_to_chr(enum common_sampler_type cnstr);
+std::string common_sampler_type_to_str(enum common_sampler_type cnstr);
+
+std::vector<enum common_sampler_type> common_sampler_types_from_names(const std::vector<std::string> & names, bool allow_alt_names);
+std::vector<enum common_sampler_type> common_sampler_types_from_chars(const std::string & chars);
+
+llama_sampler * llama_sampler_init_llg(const llama_vocab * vocab,
+                const char * grammar_kind, const char * grammar_data);