@fugood/llama.node 0.6.3 → 1.0.0-beta.2

package/src/llama.cpp/examples/speculative-simple/speculative-simple.cpp

@@ -1,261 +0,0 @@
-#include "arg.h"
-#include "common.h"
-#include "sampling.h"
-#include "speculative.h"
-#include "log.h"
-#include "llama.h"
-
-#include <cstdio>
-#include <cstring>
-#include <string>
-#include <vector>
-
-int main(int argc, char ** argv) {
-    common_params params;
-
-    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SPECULATIVE)) {
-        return 1;
-    }
-
-    if (params.n_predict < -1) {
-        LOG_ERR("%s: --n-predict must be >= -1\n", __func__);
-        return 1;
-    }
-
-    common_init();
-
-    if (params.speculative.model.path.empty()) {
-        LOG_ERR("%s: --model-draft is required\n", __func__);
-        return 1;
-    }
-
-    // init llama.cpp
-    llama_backend_init();
-    llama_numa_init(params.numa);
-
-    llama_model * model_tgt = NULL;
-    //llama_model * model_dft = NULL;
-
-    llama_context * ctx_tgt = NULL;
-    llama_context * ctx_dft = NULL;
-
-    // load the target model
-    common_init_result llama_init_tgt = common_init_from_params(params);
-
-    model_tgt = llama_init_tgt.model.get();
-    ctx_tgt   = llama_init_tgt.context.get();
-
-    const llama_vocab * vocab = llama_model_get_vocab(model_tgt);
-
-    // load the draft model
-    params.devices      = params.speculative.devices;
-    params.model        = params.speculative.model;
-    params.n_ctx        = params.speculative.n_ctx;
-    params.n_batch      = params.speculative.n_ctx > 0 ? params.speculative.n_ctx : params.n_batch;
-    params.n_gpu_layers = params.speculative.n_gpu_layers;
-
-    if (params.speculative.cpuparams.n_threads > 0) {
-        params.cpuparams.n_threads = params.speculative.cpuparams.n_threads;
-    }
-
-    params.cpuparams_batch.n_threads = params.speculative.cpuparams_batch.n_threads;
-    common_init_result llama_init_dft = common_init_from_params(params);
-
-    //model_dft = llama_init_dft.model.get();
-    ctx_dft   = llama_init_dft.context.get();
-
-    if (!common_speculative_are_compatible(ctx_tgt, ctx_dft)) {
-        return 1;
-    }
-
-    // Tokenize the prompt
-    std::vector<llama_token> inp;
-    inp = common_tokenize(ctx_tgt, params.prompt, true, true);
-
-    if (llama_n_ctx(ctx_tgt) < (uint32_t) inp.size()) {
-        LOG_ERR("%s: the prompt exceeds the context size (%d tokens, ctx %d)\n", __func__, (int) inp.size(), llama_n_ctx(ctx_tgt));
-
-        return 1;
-    }
-
-    if (llama_n_batch(ctx_tgt) < (uint32_t) inp.size()) {
-        LOG_ERR("%s: the prompt exceeds the batch size (%d tokens, batch %d)\n", __func__, (int) inp.size(), llama_n_batch(ctx_tgt));
-
-        return 1;
-    }
-
-    LOG("\n\n");
-
-    for (auto id : inp) {
-        LOG("%s", common_token_to_piece(ctx_tgt, id).c_str());
-    }
-
-    // how many tokens to draft each time
-    int n_draft     = params.speculative.n_max;
-    int n_draft_min = params.speculative.n_min;
-
-    float p_min = params.speculative.p_min;
-
-    int n_predict = 0;
-    int n_drafted = 0;
-    int n_accept  = 0;
-
-    // used to determine end of generation
-    bool has_eos = false;
-
-    // ================================================
-    // everything until here is standard initialization
-    // the relevant stuff for speculative decoding starts here
-
-    const auto t_enc_start = ggml_time_us();
-
-    // target model sampling context
-    struct common_sampler * smpl = common_sampler_init(model_tgt, params.sampling);
-
-    // eval the prompt
-    llama_decode(ctx_tgt, llama_batch_get_one(inp.data(), inp.size() - 1));
-
-    // note: keep the last token separate!
-    llama_token id_last = inp.back();
-
-    // all tokens currently in the target context
-    llama_tokens prompt_tgt(inp.begin(), inp.end() - 1);
-    prompt_tgt.reserve(llama_n_ctx(ctx_tgt));
-
-    int n_past = inp.size() - 1;
-
-    // init the speculator
-    struct common_speculative_params params_spec;
-    params_spec.n_draft = n_draft;
-    params_spec.n_reuse = llama_n_ctx(ctx_dft) - n_draft;
-    params_spec.p_min   = p_min;
-
-    struct common_speculative * spec = common_speculative_init(ctx_dft);
-
-    llama_batch batch_tgt = llama_batch_init(llama_n_batch(ctx_tgt), 0, 1);
-
-    const auto t_enc_end = ggml_time_us();
-
-    const auto t_dec_start = ggml_time_us();
-
-    while (true) {
-        // optionally, generate draft tokens that can be appended to the target batch
-        //
-        // this is the most important part of the speculation. the more probable tokens that are provided here
-        // the better the performance will be. in theory, this computation can be performed asynchronously and even
-        // offloaded to a remote device. it doesn't even have to be based on an LLM. instead, it can provide tokens
-        // from a cache or lookup tables.
-        //
-        llama_tokens draft = common_speculative_gen_draft(spec, params_spec, prompt_tgt, id_last);
-
-        //LOG_DBG("draft: %s\n", string_from(ctx_dft, draft).c_str());
-
-        // always have a token to evaluate from before - id_last
-        common_batch_clear(batch_tgt);
-        common_batch_add  (batch_tgt, id_last, n_past++, { 0 }, true);
-
-        // evaluate the target model on [id_last, draft0, draft1, ..., draftN-1]
-        {
-            // do not waste time on small drafts
-            if (draft.size() < (size_t) n_draft_min) {
-                draft.clear();
-            }
-
-            for (size_t i = 0; i < draft.size(); ++i) {
-                common_batch_add(batch_tgt, draft[i], n_past + i, { 0 }, true);
-            }
-
-            //LOG_DBG("target batch: %s\n", string_from(ctx_tgt, batch_tgt).c_str());
-
-            llama_decode(ctx_tgt, batch_tgt);
-        }
-
-        // sample from the full target batch and return the accepted tokens based on the target sampler
-        //
-        // for each token to be accepted, the sampler would have to sample that same token
-        // in such cases, instead of decoding the sampled token as we normally do, we simply continue with the
-        // available logits from the batch and sample the next token until we run out of logits or the sampler
-        // disagrees with the draft
-        //
-        const auto ids = common_sampler_sample_and_accept_n(smpl, ctx_tgt, draft);
-
-        //LOG_DBG("ids: %s\n", string_from(ctx_tgt, ids).c_str());
-
-        GGML_ASSERT(ids.size() > 0); // there will always be at least one accepted token
-
-        n_past    += ids.size() - 1;
-        n_drafted += draft.size(); // note: we ignore the discarded small drafts
-        n_accept  += ids.size() - 1;
-        n_predict += ids.size();
-
-        // process the accepted tokens and update contexts
-        //
-        // this is the standard token post-processing that we normally do
-        // in this case, we do it for a group of accepted tokens at once
-        //
-        for (size_t i = 0; i < ids.size(); ++i) {
-            prompt_tgt.push_back(id_last);
-
-            id_last = ids[i];
-
-            if (llama_vocab_is_eog(vocab, id_last)) {
-                has_eos = true;
-                break;
-            }
-
-            const std::string token_str = common_token_to_piece(ctx_tgt, id_last);
-
-            if (params.use_color && i + 1 < ids.size()) {
-                LOG("\u001b[%dm%s\u001b[37m", (36 - 0 % 6), token_str.c_str());
-            } else {
-                LOG("%s", token_str.c_str());
-            }
-        }
-
-        LOG_DBG("accepted %d/%d draft tokens, the last target token is: (%d)\n", (int) ids.size() - 1, (int) draft.size(), id_last);
-
-        {
-            LOG_DBG("clear kv cache from any extra tokens, n_past = %d\n", n_past);
-
-            llama_kv_self_seq_rm(ctx_tgt, 0, n_past, -1);
-        }
-
-        if ((params.n_predict >= 0 && n_predict > params.n_predict) || has_eos) {
-            break;
-        }
-    }
-
-    auto t_dec_end = ggml_time_us();
-
-    const int n_input = inp.size();
-
-    LOG("\n\n");
-
-    LOG_INF("encoded %4d tokens in %8.3f seconds, speed: %8.3f t/s\n", n_input,   (t_enc_end - t_enc_start) / 1e6f, inp.size() / ((t_enc_end - t_enc_start) / 1e6f));
-    LOG_INF("decoded %4d tokens in %8.3f seconds, speed: %8.3f t/s\n", n_predict, (t_dec_end - t_dec_start) / 1e6f, n_predict  / ((t_dec_end - t_dec_start) / 1e6f));
-
-    LOG_INF("\n");
-    LOG_INF("n_draft   = %d\n", n_draft);
-    LOG_INF("n_predict = %d\n", n_predict);
-    LOG_INF("n_drafted = %d\n", n_drafted);
-    LOG_INF("n_accept  = %d\n", n_accept);
-    LOG_INF("accept    = %.3f%%\n", 100.0f * n_accept / n_drafted);
-
-    LOG_INF("\n");
-    LOG_INF("draft:\n\n");
-
-    llama_perf_context_print(ctx_dft);
-
-    LOG_INF("\n");
-    LOG_INF("target:\n\n");
-    common_perf_print(ctx_tgt, smpl);
-
-    common_sampler_free(smpl);
-    common_speculative_free(spec);
-
-    llama_backend_free();
-
-    LOG("\n\n");
-
-    return 0;
-}

package/src/llama.cpp/examples/sycl/CMakeLists.txt

@@ -1,9 +0,0 @@
-#  MIT license
-#  Copyright (C) 2024 Intel Corporation
-#  SPDX-License-Identifier: MIT
-
-set(TARGET llama-ls-sycl-device)
-add_executable(${TARGET} ls-sycl-device.cpp)
-install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
-target_compile_features(${TARGET} PRIVATE cxx_std_17)

package/src/llama.cpp/examples/sycl/build.sh

@@ -1,23 +0,0 @@
-
-#  MIT license
-#  Copyright (C) 2024 Intel Corporation
-#  SPDX-License-Identifier: MIT
-
-mkdir -p build
-cd build
-source /opt/intel/oneapi/setvars.sh
-
-#for FP16
-#cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON -DLLAMA_CURL=OFF # faster for long-prompt inference
-
-#for FP32
-cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_CURL=OFF
-
-#build example/main
-#cmake --build . --config Release --target main
-
-#build example/llama-bench
-#cmake --build . --config Release --target llama-bench
-
-#build all binary
-cmake --build . --config Release -j -v

package/src/llama.cpp/examples/sycl/ls-sycl-device.cpp

@@ -1,13 +0,0 @@
-//
-//  MIT license
-//  Copyright (C) 2024 Intel Corporation
-//  SPDX-License-Identifier: MIT
-//
-
-
-#include "ggml-sycl.h"
-
-int main() {
-    ggml_backend_sycl_print_sycl_devices();
-    return 0;
-}

package/src/llama.cpp/examples/sycl/run-llama2.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-
-#  MIT license
-#  Copyright (C) 2024 Intel Corporation
-#  SPDX-License-Identifier: MIT
-export ONEAPI_DEVICE_SELECTOR="level_zero:0"
-source /opt/intel/oneapi/setvars.sh
-
-#export GGML_SYCL_DEBUG=1
-
-#ZES_ENABLE_SYSMAN=1, Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory. Recommended to use when --split-mode = layer.
-
-INPUT_PROMPT="Building a website can be done in 10 simple steps:\nStep 1:"
-MODEL_FILE=models/llama-2-7b.Q4_0.gguf
-NGL=99
-CONTEXT=4096
-
-if [ $# -gt 0 ]; then
-    GGML_SYCL_DEVICE=$1
-    echo "use $GGML_SYCL_DEVICE as main GPU"
-    #use signle GPU only
-    ZES_ENABLE_SYSMAN=1 ./build/bin/llama-cli -m ${MODEL_FILE} -p "${INPUT_PROMPT}" -n 400 -e -ngl ${NGL} -s 0 -c ${CONTEXT} -mg $GGML_SYCL_DEVICE -sm none
-
-else
-    #use multiple GPUs with same max compute units
-    ZES_ENABLE_SYSMAN=1 ./build/bin/llama-cli -m ${MODEL_FILE} -p "${INPUT_PROMPT}" -n 400 -e -ngl ${NGL} -s 0 -c ${CONTEXT}
-fi

package/src/llama.cpp/examples/sycl/run-llama3.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-
-#  MIT license
-#  Copyright (C) 2025 Intel Corporation
-#  SPDX-License-Identifier: MIT
-
-# If you want more control, DPC++ Allows selecting a specific device through the
-# following environment variable
-#export ONEAPI_DEVICE_SELECTOR="level_zero:0"
-source /opt/intel/oneapi/setvars.sh
-
-#export GGML_SYCL_DEBUG=1
-
-#ZES_ENABLE_SYSMAN=1, Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory. Recommended to use when --split-mode = layer.
-
-INPUT_PROMPT="Building a website can be done in 10 simple steps:\nStep 1:"
-MODEL_FILE=models/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf
-NGL=99 # Layers offloaded to the GPU. If the device runs out of memory, reduce this value according to the model you are using.
-CONTEXT=4096
-
-if [ $# -gt 0 ]; then
-    GGML_SYCL_DEVICE=$1
-    echo "Using $GGML_SYCL_DEVICE as the main GPU"
-    ZES_ENABLE_SYSMAN=1 ./build/bin/llama-cli -m ${MODEL_FILE} -p "${INPUT_PROMPT}" -n 400 -e -ngl ${NGL} -c ${CONTEXT} -mg $GGML_SYCL_DEVICE -sm none
-else
-    #use multiple GPUs with same max compute units
-    ZES_ENABLE_SYSMAN=1 ./build/bin/llama-cli -m ${MODEL_FILE} -p "${INPUT_PROMPT}" -n 400 -e -ngl ${NGL} -c ${CONTEXT}
-fi

package/src/llama.cpp/examples/sycl/win-build-sycl.bat

@@ -1,33 +0,0 @@
-
-::  MIT license
-::  Copyright (C) 2024 Intel Corporation
-::  SPDX-License-Identifier: MIT
-
-
-IF not exist build (mkdir build)
-cd build
-if %errorlevel% neq 0 goto ERROR
-
-@call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 --force
-if %errorlevel% neq 0 goto ERROR
-
-::  for FP16
-::  faster for long-prompt inference
-::  cmake -G "MinGW Makefiles" .. -DLLAMA_CURL=OFF -DGGML_SYCL=ON -DCMAKE_CXX_COMPILER=icx -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DGGML_SYCL_F16=ON
-
-::  for FP32
-cmake -G "Ninja" .. -DLLAMA_CURL=OFF -DGGML_SYCL=ON -DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=icx -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release
-if %errorlevel% neq 0 goto ERROR
-::  build example/main only
-::  make main
-
-::  build all binary
-cmake --build . -j
-if %errorlevel% neq 0 goto ERROR
-
-cd ..
-exit /B 0
-
-:ERROR
-echo comomand error: %errorlevel%
-exit /B %errorlevel%

package/src/llama.cpp/examples/sycl/win-run-llama2.bat

@@ -1,9 +0,0 @@
-::  MIT license
-::  Copyright (C) 2024 Intel Corporation
-::  SPDX-License-Identifier: MIT
-
-set INPUT2="Building a website can be done in 10 simple steps:\nStep 1:"
-@call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 --force
-
-
-.\build\bin\llama-cli.exe -m models\llama-2-7b.Q4_0.gguf -p %INPUT2% -n 400 -e -ngl 99 -s 0

package/src/llama.cpp/examples/sycl/win-run-llama3.bat

@@ -1,9 +0,0 @@
-::  MIT license
-::  Copyright (C) 2024 Intel Corporation
-::  SPDX-License-Identifier: MIT
-
-set INPUT2="Building a website can be done in 10 simple steps:\nStep 1:"
-@call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" intel64 --force
-
-
-.\build\bin\llama-cli.exe -m models\Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf -p %INPUT2% -n 400 -e -ngl 99

package/src/llama.cpp/examples/training/CMakeLists.txt

@@ -1,5 +0,0 @@
-set(TARGET llama-finetune)
-add_executable(${TARGET} finetune.cpp)
-install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
-target_compile_features(${TARGET} PRIVATE cxx_std_11)

package/src/llama.cpp/examples/training/finetune.cpp

@@ -1,96 +0,0 @@
-#include "arg.h"
-#include "common.h"
-#include "log.h"
-#include "llama.h"
-
-#include <cmath>
-#include <cstdio>
-#include <cstring>
-#include <ctime>
-#include <vector>
-
-#if defined(_MSC_VER)
-#pragma warning(disable: 4244 4267) // possible loss of data
-#endif
-
-int main(int argc, char ** argv) {
-    common_params params;
-
-    params.escape = false;
-
-    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_PERPLEXITY)) {
-        return 1;
-    }
-
-    if (params.use_mmap) {
-        LOG_INF("%s: force disabling memory mapping because it would result in-read-only pointers to the weights\n", __func__);
-        params.use_mmap = false;
-    }
-    if (params.cache_type_k != GGML_TYPE_F32) {
-        LOG_INF("%s: force changing k cache type to f32 due to a lack of f16 support for OUT_PROD\n", __func__);
-        params.cache_type_k = GGML_TYPE_F32;
-    }
-    if (params.cache_type_v != GGML_TYPE_F32) {
-        LOG_INF("%s: force changing v cache type to f32 due to a lack of f16 support for OUT_PROD\n", __func__);
-        params.cache_type_v = GGML_TYPE_F32;
-    }
-
-    common_init();
-    llama_backend_init();
-    llama_numa_init(params.numa);
-
-    // load the model and apply lora adapter, if any
-    common_init_result llama_init = common_init_from_params(params);
-    llama_model_ptr   & model = llama_init.model;
-    llama_context_ptr & ctx   = llama_init.context;
-
-    if (model == NULL) {
-        LOG_ERR("%s: unable to load model\n", __func__);
-        return 1;
-    }
-
-    // print system information
-    {
-        LOG_INF("\n");
-        LOG_INF("%s\n", common_params_get_system_info(params).c_str());
-    }
-
-    constexpr float val_split = 0.05f;
-
-    std::vector<llama_token> tokens = common_tokenize(ctx.get(), params.prompt, true);
-    ggml_opt_dataset_t dataset = common_opt_dataset_init(ctx.get(), tokens, llama_n_ctx(ctx.get())/2);
-
-    struct ggml_opt_optimizer_params optimizer_params = ggml_opt_get_default_optimizer_params(nullptr);
-    optimizer_params.adamw.alpha = 1e-7f; // learning rate
-
-    struct llama_opt_params lopt_params {
-        /*n_ctx_train     =*/ 0,
-        /*param_filter    =*/ llama_opt_param_filter_all,
-        /*param_filter_ud =*/ nullptr,
-        /*get_opt_pars    =*/ ggml_opt_get_constant_optimizer_params,
-        /*get_opt_pars_ud =*/ &optimizer_params,
-    };
-    llama_opt_init(ctx.get(), model.get(), lopt_params);
-
-    const int64_t idata_split = ggml_opt_dataset_ndata(dataset) * (1.0f - val_split);
-
-    ggml_opt_result_t result_train = ggml_opt_result_init();
-    ggml_opt_result_t result_eval  = ggml_opt_result_init();
-
-    for (int epoch = 0; epoch < 2; ++epoch) {
-        llama_opt_epoch(ctx.get(), dataset, result_train, result_eval, idata_split,
-            ggml_opt_epoch_callback_progress_bar, ggml_opt_epoch_callback_progress_bar);
-        fprintf(stderr, "\n");
-
-        ggml_opt_result_reset(result_train);
-        ggml_opt_result_reset(result_eval);
-    }
-    ggml_opt_result_free(result_train);
-    ggml_opt_result_free(result_eval);
-
-    llama_model_save_to_file(model.get(), "finetuned-model.gguf");
-
-    llama_backend_free();
-
-    return 0;
-}

package/src/llama.cpp/ggml/cmake/GitVars.cmake

@@ -1,22 +0,0 @@
-find_package(Git)
-
-# the commit's SHA1
-execute_process(COMMAND
-    "${GIT_EXECUTABLE}" describe --match=NeVeRmAtCh --always --abbrev=8
-    WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
-    OUTPUT_VARIABLE GIT_SHA1
-    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
-
-# the date of the commit
-execute_process(COMMAND
-    "${GIT_EXECUTABLE}" log -1 --format=%ad --date=local
-    WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
-    OUTPUT_VARIABLE GIT_DATE
-    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
-
-# the subject of the commit
-execute_process(COMMAND
-    "${GIT_EXECUTABLE}" log -1 --format=%s
-    WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
-    OUTPUT_VARIABLE GIT_COMMIT_SUBJECT
-    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)

package/src/llama.cpp/ggml/cmake/common.cmake

@@ -1,26 +0,0 @@
-function(ggml_get_flags CCID CCVER)
-    set(C_FLAGS "")
-    set(CXX_FLAGS "")
-
-    if (CCID MATCHES "Clang")
-        set(C_FLAGS   -Wunreachable-code-break -Wunreachable-code-return)
-        set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi)
-
-        if (
-            (CCID STREQUAL "Clang"      AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR
-            (CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0)
-        )
-            list(APPEND C_FLAGS -Wdouble-promotion)
-        endif()
-    elseif (CCID STREQUAL "GNU")
-        set(C_FLAGS   -Wdouble-promotion)
-        set(CXX_FLAGS -Wno-array-bounds)
-
-        if (CCVER VERSION_GREATER_EQUAL 8.1.0)
-            list(APPEND CXX_FLAGS -Wextra-semi)
-        endif()
-    endif()
-
-    set(GF_C_FLAGS   ${C_FLAGS}   PARENT_SCOPE)
-    set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE)
-endfunction()