@fugood/llama.node 0.6.3 → 1.0.0-beta.1

package/src/llama.cpp/examples/passkey/passkey.cpp

@@ -1,277 +0,0 @@
-#include "arg.h"
-#include "common.h"
-#include "log.h"
-#include "llama.h"
-
-#include <cmath>
-#include <cstdio>
-#include <string>
-#include <vector>
-#include <algorithm>
-
-static void print_usage(int, char ** argv) {
-    LOG("\nexample usage:\n");
-    LOG("\n    %s -m model.gguf --junk 250 --pos 90 --keep 32 --grp-attn-n 2 [--seed 1234]\n", argv[0]);
-    LOG("\n");
-}
-
-int main(int argc, char ** argv) {
-    common_params params;
-
-    params.n_junk = 250;
-    params.n_keep = 32;
-    params.i_pos  = -1;
-
-    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_PASSKEY, print_usage)) {
-        return 1;
-    }
-
-    common_init();
-
-    int n_junk = params.n_junk;
-    int n_keep = params.n_keep;
-    int n_grp  = params.grp_attn_n;
-    int i_pos  = params.i_pos;
-
-    if (i_pos == -1) {
-        i_pos = rand() % n_junk;
-    }
-
-    const std::string prompt_prefix = "There is an important info hidden inside a lot of irrelevant text. Find it and memorize them. I will quiz you about the important information there.";
-    const std::string prompt_suffix = " What is the pass key? The pass key is";
-
-    // generate junk text
-    params.prompt = prompt_prefix;
-
-    const int passkey = rand() % 50000 + 1;
-
-    for (int i = 0; i < n_junk; i++) {
-        if (i % n_junk == i_pos) {
-            params.prompt += " The pass key is " + std::to_string(passkey) + ". Remember it. " + std::to_string(passkey) + " is the pass key.";
-        }
-
-        params.prompt += " The grass is green. The sky is blue. The sun is yellow. Here we go. There and back again.";
-    }
-
-    params.prompt += prompt_suffix;
-
-    // init LLM
-
-    llama_backend_init();
-    llama_numa_init(params.numa);
-
-    // initialize the model
-
-    llama_model_params model_params = common_model_params_to_llama(params);
-
-    llama_model * model = llama_model_load_from_file(params.model.path.c_str(), model_params);
-
-    if (model == NULL) {
-        LOG_ERR("%s: unable to load model\n" , __func__);
-        return 1;
-    }
-
-    const llama_vocab * vocab = llama_model_get_vocab(model);
-
-    // initialize the context
-
-    llama_context_params ctx_params = common_context_params_to_llama(params);
-
-    ctx_params.n_ctx = llama_model_n_ctx_train(model)*n_grp + n_keep;
-
-    GGML_ASSERT(ctx_params.n_batch % n_grp == 0 && "n_batch must be divisible by n_grp");
-
-    llama_context * ctx = llama_init_from_model(model, ctx_params);
-    if (ctx == NULL) {
-        LOG_ERR("%s: failed to create the llama_context\n" , __func__);
-        return 1;
-    }
-
-    auto sparams = llama_sampler_chain_default_params();
-
-    llama_sampler * smpl = llama_sampler_chain_init(sparams);
-
-    llama_sampler_chain_add(smpl, llama_sampler_init_greedy());
-
-    // tokenize the prompt
-    std::vector<llama_token> tokens_list;
-    tokens_list = common_tokenize(ctx, params.prompt, true);
-
-    // tokenize the prefix and use it as a sink
-    const int n_tokens_prefix = common_tokenize(ctx, prompt_prefix, true).size();
-
-    const int n_tokens_all = tokens_list.size();
-
-    // we leave a margin of 16 tokens for the generated text - it should contain just the passkey
-    const int n_predict = 16;
-
-    // total length of the sequences including the prompt
-    const int n_len = n_tokens_all + n_predict;
-
-    const int n_ctx       = llama_n_ctx(ctx) - n_keep;
-    const int n_kv_req    = llama_n_ctx(ctx);
-    const int n_batch     = ctx_params.n_batch;
-    const int n_batch_grp = ctx_params.n_batch/n_grp;
-
-    LOG_INF("\n%s: n_len = %d, n_ctx = %d, n_kv_req = %d, n_grp = %d, n_batch = %d, n_junk = %d, i_pos = %d\n", __func__, n_len, n_ctx, n_kv_req, n_grp, n_batch, n_junk, i_pos);
-
-    // print the prompt token-by-token
-
-    LOG_INF("\n");
-    LOG_INF("prefix tokens: %d\n", n_tokens_prefix);
-    LOG_INF("prompt tokens: %d\n", n_tokens_all);
-    //LOG_INF("prompt: %s\n", params.prompt.c_str());
-
-    llama_batch batch = llama_batch_init(params.n_batch, 0, 1);
-
-    int n_past = 0;
-
-    // fill the KV cache
-    for (int i = 0; i < n_ctx; i += n_batch) {
-        if (i > 0 && n_grp > 1) {
-            // if SelfExtend is enabled, we compress the position from the last batch by a factor of n_grp
-            const int ib = i/n_batch - 1;
-            const int bd = n_batch_grp*(n_grp - 1);
-
-            llama_kv_self_seq_add (ctx, 0, n_past - n_batch,         n_past,         ib*bd);
-            llama_kv_self_seq_div (ctx, 0, n_past - n_batch + ib*bd, n_past + ib*bd, n_grp);
-            llama_kv_self_update  (ctx);
-
-            n_past = llama_kv_self_seq_pos_max(ctx, 0) + 1;
-        }
-
-        common_batch_clear(batch);
-
-        for (int j = 0; j < n_batch && i + j < n_tokens_all; j++) {
-            common_batch_add(batch, tokens_list[i + j], n_past++, { 0 }, false);
-        }
-
-        if (i + n_batch >= n_tokens_all) {
-            batch.logits[batch.n_tokens - 1] = true;
-        }
-
-        if (llama_decode(ctx, batch) != 0) {
-            LOG_INF("%s: llama_decode() failed\n", __func__);
-            return 1;
-        }
-
-        LOG_INF("%s: processed: [%6d, %6d)\n", __func__, i, std::min(i + n_batch, n_tokens_all));
-
-        if (i + n_batch >= n_tokens_all) {
-            break;
-        }
-    }
-
-    for (int i = n_ctx; i < n_tokens_all; i += n_batch) {
-        const int n_discard = n_batch;
-
-        LOG_INF("%s: shifting KV cache with %d\n", __func__, n_discard);
-
-        llama_kv_self_seq_rm (ctx, 0, n_keep            , n_keep + n_discard);
-        llama_kv_self_seq_add(ctx, 0, n_keep + n_discard, n_ctx,  -n_discard);
-      //llama_kv_self_defrag (ctx);
-        llama_kv_self_update (ctx);
-
-        n_past = llama_kv_self_seq_pos_max(ctx, 0) + 1;
-
-        common_batch_clear(batch);
-
-        for (int j = 0; j < n_batch && i + j < n_tokens_all; j++) {
-            common_batch_add(batch, tokens_list[i + j], n_past++, { 0 }, false);
-        }
-
-        if (i + n_batch >= n_tokens_all) {
-            batch.logits[batch.n_tokens - 1] = true;
-        }
-
-        if (llama_decode(ctx, batch) != 0) {
-            LOG_ERR("%s: llama_decode() failed\n", __func__);
-            return 1;
-        }
-
-        LOG_INF("%s: processed: [%6d, %6d)\n", __func__, i, std::min(i + n_batch, n_tokens_all));
-    }
-
-    {
-        const int n_discard = n_past - n_ctx + n_predict;
-
-        if (n_discard > 0) {
-            LOG_INF("%s: shifting KV cache with %d to free space for the answer\n", __func__, n_discard);
-
-            llama_kv_self_seq_rm (ctx, 0, n_keep            , n_keep + n_discard);
-            llama_kv_self_seq_add(ctx, 0, n_keep + n_discard, n_ctx,  -n_discard);
-          //llama_kv_self_defrag (ctx);
-            llama_kv_self_update (ctx);
-
-            n_past = llama_kv_self_seq_pos_max(ctx, 0) + 1;
-        }
-    }
-
-    LOG_INF("\n");
-    LOG_INF("%s: passkey = %d, inserted at position %d / %d (token pos: ~%d)\n", __func__, passkey, i_pos, n_junk, (i_pos * n_tokens_all) / n_junk);
-    LOG_INF("\n");
-
-    // main loop
-
-    int n_cur    = n_tokens_all;
-    int n_decode = 0;
-
-    LOG_INF("%s", prompt_suffix.c_str());
-
-    const auto t_main_start = ggml_time_us();
-
-    while (n_cur <= n_len) {
-        // sample the next token
-        {
-            const llama_token new_token_id = llama_sampler_sample(smpl, ctx, batch.n_tokens - 1);
-
-            // is it an end of generation?
-            if (llama_vocab_is_eog(vocab, new_token_id) || n_cur == n_len) {
-                LOG("\n");
-
-                break;
-            }
-
-            LOG("%s", common_token_to_piece(ctx, new_token_id).c_str());
-
-            n_decode += 1;
-
-            // prepare the next batch
-            common_batch_clear(batch);
-
-            // push this new token for next evaluation
-            common_batch_add(batch, new_token_id, n_past++, { 0 }, true);
-        }
-
-        n_cur += 1;
-
-        // evaluate the current batch with the transformer model
-        if (llama_decode(ctx, batch)) {
-            LOG_ERR("%s : failed to eval, return code %d\n", __func__, 1);
-            return 1;
-        }
-    }
-
-    LOG("\n");
-
-    const auto t_main_end = ggml_time_us();
-
-    LOG_INF("%s: decoded %d tokens in %.2f s, speed: %.2f t/s\n",
-            __func__, n_decode, (t_main_end - t_main_start) / 1000000.0f, n_decode / ((t_main_end - t_main_start) / 1000000.0f));
-
-    LOG("\n");
-    llama_perf_context_print(ctx);
-
-    LOG("\n");
-
-    llama_sampler_free(smpl);
-
-    llama_batch_free(batch);
-
-    llama_free(ctx);
-    llama_model_free(model);
-
-    llama_backend_free();
-
-    return 0;
-}

package/src/llama.cpp/examples/retrieval/CMakeLists.txt

@@ -1,5 +0,0 @@
-set(TARGET llama-retrieval)
-add_executable(${TARGET} retrieval.cpp)
-install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
-target_compile_features(${TARGET} PRIVATE cxx_std_17)

package/src/llama.cpp/examples/retrieval/retrieval.cpp

@@ -1,304 +0,0 @@
-#include "arg.h"
-#include "common.h"
-#include "log.h"
-#include "llama.h"
-
-#include <algorithm>
-#include <fstream>
-#include <iostream> // TODO: remove me
-
-static void print_usage(int, char ** argv) {
-    LOG("\nexample usage:\n");
-    LOG("\n    %s --model ./models/bge-base-en-v1.5-f16.gguf --top-k 3 --context-file README.md --context-file License --chunk-size 100 --chunk-separator .\n", argv[0]);
-    LOG("\n");
-}
-
-struct chunk {
-    // filename
-    std::string filename;
-    // original file position
-    size_t filepos;
-    // original text data
-    std::string textdata;
-    // tokenized text data
-    std::vector<llama_token> tokens;
-    // embedding
-    std::vector<float> embedding;
-};
-
-// chunk file data to chunks of size >= chunk_size
-// chunk_separator is the separator between chunks
-static std::vector<chunk> chunk_file(const std::string & filename, int chunk_size, const std::string & chunk_separator) {
-    std::vector<chunk> chunks;
-    std::ifstream f(filename.c_str());
-
-    if (!f.is_open()) {
-        LOG_ERR("could not open file %s\n", filename.c_str());
-        return chunks;
-    }
-
-    chunk current_chunk;
-    char buffer[1024];
-    int64_t filepos = 0;
-    std::string current;
-    while (f.read(buffer, 1024)) {
-        current += std::string(buffer, f.gcount());
-        size_t pos;
-        while ((pos = current.find(chunk_separator)) != std::string::npos) {
-            current_chunk.textdata += current.substr(0, pos + chunk_separator.size());
-            if ((int) current_chunk.textdata.size() > chunk_size) {
-                // save chunk
-                current_chunk.filepos = filepos;
-                current_chunk.filename = filename;
-                chunks.push_back(current_chunk);
-                // update filepos
-                filepos += (int) current_chunk.textdata.size();
-                // reset current_chunk
-                current_chunk = chunk();
-            }
-            current = current.substr(pos + chunk_separator.size());
-        }
-
-    }
-    // add leftover data to last chunk
-    if (current_chunk.textdata.size() > 0) {
-        if (chunks.empty()) {
-            current_chunk.filepos = filepos;
-            current_chunk.filename = filename;
-            chunks.push_back(current_chunk);
-        } else {
-            chunks.back().textdata += current_chunk.textdata;
-        }
-    }
-    f.close();
-    return chunks;
-}
-
-static void batch_add_seq(llama_batch & batch, const std::vector<int32_t> & tokens, llama_seq_id seq_id) {
-    size_t n_tokens = tokens.size();
-    for (size_t i = 0; i < n_tokens; i++) {
-        common_batch_add(batch, tokens[i], i, { seq_id }, true);
-    }
-}
-
-static void batch_encode(llama_context * ctx, llama_batch & batch, float * output, int n_seq, int n_embd) {
-    // clear previous kv_cache values (irrelevant for embeddings)
-    llama_kv_self_clear(ctx);
-
-    // run model
-    LOG_INF("%s: n_tokens = %d, n_seq = %d\n", __func__, batch.n_tokens, n_seq);
-    if (llama_encode(ctx, batch) < 0) {
-        LOG_ERR("%s : failed to encode\n", __func__);
-    }
-
-    for (int i = 0; i < batch.n_tokens; i++) {
-        if (!batch.logits[i]) {
-            continue;
-        }
-
-        // try to get sequence embeddings - supported only when pooling_type is not NONE
-        const float * embd = llama_get_embeddings_seq(ctx, batch.seq_id[i][0]);
-        if (embd == NULL) {
-            embd = llama_get_embeddings_ith(ctx, i);
-            if (embd == NULL) {
-                LOG_ERR("%s: failed to get embeddings for token %d\n", __func__, i);
-                continue;
-            }
-        }
-
-        float * out = output + batch.seq_id[i][0] * n_embd;
-        common_embd_normalize(embd, out, n_embd, 2);
-    }
-}
-
-int main(int argc, char ** argv) {
-    common_params params;
-
-    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_RETRIEVAL, print_usage)) {
-        return 1;
-    }
-
-    common_init();
-
-    // For BERT models, batch size must be equal to ubatch size
-    params.n_ubatch = params.n_batch;
-    params.embedding = true;
-
-    if (params.chunk_size <= 0) {
-        LOG_ERR("chunk_size must be positive\n");
-        return 1;
-    }
-    if (params.context_files.empty()) {
-        LOG_ERR("context_files must be specified\n");
-        return 1;
-    }
-
-    LOG_INF("processing files:\n");
-    for (auto & context_file : params.context_files) {
-        LOG_INF("%s\n", context_file.c_str());
-    }
-
-    std::vector<chunk> chunks;
-    for (auto & context_file : params.context_files) {
-        std::vector<chunk> file_chunk = chunk_file(context_file, params.chunk_size, params.chunk_separator);
-        chunks.insert(chunks.end(), file_chunk.begin(), file_chunk.end());
-    }
-    LOG_INF("Number of chunks: %zu\n", chunks.size());
-
-    llama_backend_init();
-    llama_numa_init(params.numa);
-
-    // load the model
-    common_init_result llama_init = common_init_from_params(params);
-
-    llama_model * model = llama_init.model.get();
-    llama_context * ctx = llama_init.context.get();
-
-    if (model == NULL) {
-        LOG_ERR("%s: unable to load model\n", __func__);
-        return 1;
-    }
-
-    const llama_vocab * vocab = llama_model_get_vocab(model);
-
-    const int n_ctx_train = llama_model_n_ctx_train(model);
-    const int n_ctx = llama_n_ctx(ctx);
-
-    const enum llama_pooling_type pooling_type = llama_pooling_type(ctx);
-    if (pooling_type == LLAMA_POOLING_TYPE_NONE) {
-        LOG_ERR("%s: pooling type NONE not supported\n", __func__);
-        return 1;
-    }
-
-    if (n_ctx > n_ctx_train) {
-        LOG_WRN("%s: warning: model was trained on only %d context tokens (%d specified)\n",
-                __func__, n_ctx_train, n_ctx);
-    }
-
-    // print system information
-    {
-        LOG_INF("\n");
-        LOG_INF("%s\n", common_params_get_system_info(params).c_str());
-    }
-
-    // max batch size
-    const uint64_t n_batch = params.n_batch;
-    GGML_ASSERT(params.n_batch >= params.n_ctx);
-
-    // tokenize the prompts and trim
-    for (auto & chunk : chunks) {
-        auto inp = common_tokenize(ctx, chunk.textdata, true, false);
-        if (inp.size() > n_batch) {
-            LOG_ERR("%s: chunk size (%lld) exceeds batch size (%lld), increase batch size and re-run\n",
-                    __func__, (long long int) inp.size(), (long long int) n_batch);
-            return 1;
-        }
-        // add eos if not present
-        if (llama_vocab_eos(vocab) >= 0 && (inp.empty() || inp.back() != llama_vocab_eos(vocab))) {
-            inp.push_back(llama_vocab_eos(vocab));
-        }
-        chunk.tokens = inp;
-    }
-
-    // tokenization stats
-    if (params.verbose_prompt) {
-        for (int i = 0; i < (int) chunks.size(); i++) {
-            LOG_INF("%s: prompt %d: '%s'\n", __func__, i, chunks[i].textdata.c_str());
-            LOG_INF("%s: number of tokens in prompt = %zu\n", __func__, chunks[i].tokens.size());
-            for (int j = 0; j < (int) chunks[i].tokens.size(); j++) {
-                LOG_INF("%6d -> '%s'\n", chunks[i].tokens[j], common_token_to_piece(ctx, chunks[i].tokens[j]).c_str());
-            }
-            LOG_INF("\n\n");
-        }
-    }
-
-    // initialize batch
-    const int n_chunks = chunks.size();
-    struct llama_batch batch = llama_batch_init(n_batch, 0, 1);
-
-    // allocate output
-    const int n_embd = llama_model_n_embd(model);
-    std::vector<float> embeddings(n_chunks * n_embd, 0);
-    float * emb = embeddings.data();
-
-    // break into batches
-    int p = 0; // number of prompts processed already
-    int s = 0; // number of prompts in current batch
-    for (int k = 0; k < n_chunks; k++) {
-        // clamp to n_batch tokens
-        auto & inp = chunks[k].tokens;
-
-        const uint64_t n_toks = inp.size();
-
-        // encode if at capacity
-        if (batch.n_tokens + n_toks > n_batch) {
-            float * out = emb + p * n_embd;
-            batch_encode(ctx, batch, out, s, n_embd);
-            common_batch_clear(batch);
-            p += s;
-            s = 0;
-        }
-
-        // add to batch
-        batch_add_seq(batch, inp, s);
-        s += 1;
-    }
-
-    // final batch
-    float * out = emb + p * n_embd;
-    batch_encode(ctx, batch, out, s, n_embd);
-
-    // save embeddings to chunks
-    for (int i = 0; i < n_chunks; i++) {
-        chunks[i].embedding = std::vector<float>(emb + i * n_embd, emb + (i + 1) * n_embd);
-        // clear tokens as they are no longer needed
-        chunks[i].tokens.clear();
-    }
-
-    struct llama_batch query_batch = llama_batch_init(n_batch, 0, 1);
-
-    // start loop, receive query and return top k similar chunks based on cosine similarity
-    std::string query;
-    while (true) {
-        LOG("Enter query: ");
-        std::getline(std::cin, query);
-        std::vector<int32_t> query_tokens = common_tokenize(ctx, query, true);
-
-        batch_add_seq(query_batch, query_tokens, 0);
-
-        std::vector<float> query_emb(n_embd, 0);
-        batch_encode(ctx, query_batch, query_emb.data(), 1, n_embd);
-
-        common_batch_clear(query_batch);
-
-        // compute cosine similarities
-        {
-            std::vector<std::pair<int, float>> similarities;
-            for (int i = 0; i < n_chunks; i++) {
-                float sim = common_embd_similarity_cos(chunks[i].embedding.data(), query_emb.data(), n_embd);
-                similarities.push_back(std::make_pair(i, sim));
-            }
-
-            // sort similarities
-            std::sort(similarities.begin(), similarities.end(), [](const std::pair<int, float> & a, const std::pair<int, float> & b) {
-                return a.second > b.second;
-            });
-
-            LOG("Top %d similar chunks:\n", params.sampling.top_k);
-            for (int i = 0; i < std::min(params.sampling.top_k, (int) chunks.size()); i++) {
-                LOG("filename: %s\n", chunks[similarities[i].first].filename.c_str());
-                LOG("filepos: %lld\n", (long long int) chunks[similarities[i].first].filepos);
-                LOG("similarity: %f\n", similarities[i].second);
-                LOG("textdata:\n%s\n", chunks[similarities[i].first].textdata.c_str());
-                LOG("--------------------\n");
-            }
-        }
-    }
-
-    LOG("\n");
-    llama_perf_context_print(ctx);
-
-    // clean up
-    llama_batch_free(query_batch);
-    llama_backend_free();
-}

package/src/llama.cpp/examples/save-load-state/CMakeLists.txt

@@ -1,5 +0,0 @@
-set(TARGET llama-save-load-state)
-add_executable(${TARGET} save-load-state.cpp)
-install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
-target_compile_features(${TARGET} PRIVATE cxx_std_17)