llama_cpp 0.15.4 → 0.16.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/ext/llama_cpp/extconf.rb +3 -2
- data/ext/llama_cpp/llama_cpp.cpp +17 -3
- data/lib/llama_cpp/version.rb +2 -2
- data/sig/llama_cpp.rbs +15 -1
- data/vendor/tmp/llama.cpp/Makefile +166 -82
- data/vendor/tmp/llama.cpp/ggml-alloc.c +82 -26
- data/vendor/tmp/llama.cpp/ggml-backend-impl.h +20 -8
- data/vendor/tmp/llama.cpp/ggml-backend.c +183 -69
- data/vendor/tmp/llama.cpp/ggml-backend.h +4 -4
- data/vendor/tmp/llama.cpp/ggml-blas.cpp +363 -0
- data/vendor/tmp/llama.cpp/ggml-blas.h +23 -0
- data/vendor/tmp/llama.cpp/ggml-common.h +6 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/acc.cu +47 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/arange.cu +34 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/argsort.cu +104 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/binbcast.cu +280 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/clamp.cu +34 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/concat.cu +196 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/convert.cu +686 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/cpy.cu +490 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/diagmask.cu +40 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/dmmv.cu +674 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/fattn-tile-f16.cu +319 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/fattn-tile-f32.cu +312 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/fattn.cu +345 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/getrows.cu +178 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/im2col.cu +104 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/mmq.cu +88 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/mmvq.cu +419 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/norm.cu +221 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/pad.cu +49 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/pool2d.cu +94 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/quantize.cu +112 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/rope.cu +271 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/scale.cu +31 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/softmax.cu +206 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/sumrows.cu +40 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb16.cu +10 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb32.cu +9 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb16.cu +10 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb32.cu +10 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb8.cu +8 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/tsembd.cu +47 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/unary.cu +286 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/upscale.cu +51 -0
- data/vendor/tmp/llama.cpp/ggml-cuda.cu +103 -135
- data/vendor/tmp/llama.cpp/ggml-kompute.cpp +29 -13
- data/vendor/tmp/llama.cpp/ggml-metal.h +1 -1
- data/vendor/tmp/llama.cpp/ggml-metal.m +45 -33
- data/vendor/tmp/llama.cpp/ggml-metal.metal +83 -59
- data/vendor/tmp/llama.cpp/ggml-rpc.cpp +15 -14
- data/vendor/tmp/llama.cpp/ggml-sycl.cpp +26 -90
- data/vendor/tmp/llama.cpp/ggml-vulkan-shaders.hpp +74522 -14913
- data/vendor/tmp/llama.cpp/ggml-vulkan.cpp +631 -471
- data/vendor/tmp/llama.cpp/ggml.c +278 -603
- data/vendor/tmp/llama.cpp/ggml.h +9 -28
- data/vendor/tmp/llama.cpp/llama.cpp +345 -473
- data/vendor/tmp/llama.cpp/llama.h +21 -43
- metadata +134 -7
- data/vendor/tmp/llama.cpp/ggml-mpi.c +0 -216
- data/vendor/tmp/llama.cpp/ggml-mpi.h +0 -39
- data/vendor/tmp/llama.cpp/ggml-opencl.cpp +0 -2305
- data/vendor/tmp/llama.cpp/ggml-opencl.h +0 -36
|
@@ -13,8 +13,6 @@
|
|
|
13
13
|
|
|
14
14
|
#ifdef GGML_USE_CUDA
|
|
15
15
|
# include "ggml-cuda.h"
|
|
16
|
-
#elif defined(GGML_USE_CLBLAST)
|
|
17
|
-
# include "ggml-opencl.h"
|
|
18
16
|
#elif defined(GGML_USE_VULKAN)
|
|
19
17
|
# include "ggml-vulkan.h"
|
|
20
18
|
#elif defined(GGML_USE_SYCL)
|
|
@@ -23,6 +21,10 @@
|
|
|
23
21
|
# include "ggml-kompute.h"
|
|
24
22
|
#endif
|
|
25
23
|
|
|
24
|
+
#ifdef GGML_USE_BLAS
|
|
25
|
+
# include "ggml-blas.h"
|
|
26
|
+
#endif
|
|
27
|
+
|
|
26
28
|
#ifdef GGML_USE_METAL
|
|
27
29
|
# include "ggml-metal.h"
|
|
28
30
|
#endif
|
|
@@ -110,7 +112,7 @@
|
|
|
110
112
|
//
|
|
111
113
|
|
|
112
114
|
LLAMA_ATTRIBUTE_FORMAT(2, 3)
|
|
113
|
-
static void llama_log_internal (ggml_log_level level, const char* format, ...);
|
|
115
|
+
static void llama_log_internal (ggml_log_level level, const char * format, ...);
|
|
114
116
|
static void llama_log_callback_default(ggml_log_level level, const char * text, void * user_data);
|
|
115
117
|
|
|
116
118
|
#define LLAMA_LOG_INFO(...) llama_log_internal(GGML_LOG_LEVEL_INFO , __VA_ARGS__)
|
|
@@ -706,6 +708,7 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA
|
|
|
706
708
|
{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },
|
|
707
709
|
{ LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" },
|
|
708
710
|
{ LLM_TENSOR_TOKEN_TYPES, "token_types" },
|
|
711
|
+
{ LLM_TENSOR_ATTN_NORM_2, "blk.%d.attn_norm_2" },
|
|
709
712
|
{ LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" },
|
|
710
713
|
{ LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
|
|
711
714
|
{ LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm" },
|
|
@@ -1850,7 +1853,7 @@ struct llama_hparams {
|
|
|
1850
1853
|
float rope_attn_factor = 1.0f;
|
|
1851
1854
|
float rope_freq_base_train;
|
|
1852
1855
|
float rope_freq_scale_train;
|
|
1853
|
-
uint32_t
|
|
1856
|
+
uint32_t n_ctx_orig_yarn;
|
|
1854
1857
|
float rope_yarn_log_mul;
|
|
1855
1858
|
|
|
1856
1859
|
// for State Space Models
|
|
@@ -1892,7 +1895,7 @@ struct llama_hparams {
|
|
|
1892
1895
|
if (this->n_expert_shared != other.n_expert_shared) return true;
|
|
1893
1896
|
|
|
1894
1897
|
if (this->rope_finetuned != other.rope_finetuned) return true;
|
|
1895
|
-
if (this->
|
|
1898
|
+
if (this->n_ctx_orig_yarn != other.n_ctx_orig_yarn) return true;
|
|
1896
1899
|
|
|
1897
1900
|
if (this->ssm_d_conv != other.ssm_d_conv) return true;
|
|
1898
1901
|
if (this->ssm_d_inner != other.ssm_d_inner) return true;
|
|
@@ -1951,7 +1954,7 @@ struct llama_cparams {
|
|
|
1951
1954
|
float rope_freq_base;
|
|
1952
1955
|
float rope_freq_scale;
|
|
1953
1956
|
|
|
1954
|
-
uint32_t
|
|
1957
|
+
uint32_t n_ctx_orig_yarn;
|
|
1955
1958
|
// These hyperparameters are not exposed in GGUF, because all
|
|
1956
1959
|
// existing YaRN models use the same values for them.
|
|
1957
1960
|
float yarn_ext_factor;
|
|
@@ -2149,12 +2152,12 @@ struct llama_control_vector {
|
|
|
2149
2152
|
struct llama_vocab {
|
|
2150
2153
|
using id = int32_t;
|
|
2151
2154
|
using token = std::string;
|
|
2152
|
-
using
|
|
2155
|
+
using tattr = llama_token_attr;
|
|
2153
2156
|
|
|
2154
2157
|
struct token_data {
|
|
2155
2158
|
token text;
|
|
2156
2159
|
float score;
|
|
2157
|
-
|
|
2160
|
+
tattr attr;
|
|
2158
2161
|
};
|
|
2159
2162
|
|
|
2160
2163
|
enum llama_vocab_type type = LLAMA_VOCAB_TYPE_SPM;
|
|
@@ -2164,8 +2167,7 @@ struct llama_vocab {
|
|
|
2164
2167
|
std::vector<token_data> id_to_token;
|
|
2165
2168
|
|
|
2166
2169
|
std::vector<id> cache_special_tokens;
|
|
2167
|
-
std::vector<token> cache_token_to_piece;
|
|
2168
|
-
std::vector<token> cache_token_to_piece_special; // llama_token_to_piece(special = true);
|
|
2170
|
+
std::vector<token> cache_token_to_piece; // llama_token_to_piece(special = true);
|
|
2169
2171
|
|
|
2170
2172
|
std::map<std::pair<std::string, std::string>, int> bpe_ranks;
|
|
2171
2173
|
|
|
@@ -2301,9 +2303,13 @@ struct llama_context {
|
|
|
2301
2303
|
std::vector<ggml_backend_t> backends;
|
|
2302
2304
|
#ifdef GGML_USE_METAL
|
|
2303
2305
|
ggml_backend_t backend_metal = nullptr;
|
|
2306
|
+
#endif
|
|
2307
|
+
#ifdef GGML_USE_BLAS
|
|
2308
|
+
ggml_backend_t backend_blas = nullptr;
|
|
2304
2309
|
#endif
|
|
2305
2310
|
ggml_backend_t backend_cpu = nullptr;
|
|
2306
2311
|
|
|
2312
|
+
|
|
2307
2313
|
const llama_model & model;
|
|
2308
2314
|
|
|
2309
2315
|
// key + value cache for the self attention
|
|
@@ -2372,13 +2378,34 @@ struct llama_context {
|
|
|
2372
2378
|
struct llama_control_vector cvec;
|
|
2373
2379
|
};
|
|
2374
2380
|
|
|
2381
|
+
static size_t llama_get_device_count(const llama_model & model) {
|
|
2382
|
+
size_t count = 1;
|
|
2383
|
+
#if defined(GGML_USE_CUDA)
|
|
2384
|
+
count = ggml_backend_cuda_get_device_count();
|
|
2385
|
+
#elif defined(GGML_USE_SYCL)
|
|
2386
|
+
count = ggml_backend_sycl_get_device_count();
|
|
2387
|
+
#elif defined(GGML_USE_VULKAN)
|
|
2388
|
+
count = ggml_backend_vk_get_device_count();
|
|
2389
|
+
#endif
|
|
2390
|
+
#if defined(GGML_USE_RPC)
|
|
2391
|
+
count += model.rpc_servers.size();
|
|
2392
|
+
#endif
|
|
2393
|
+
return count;
|
|
2394
|
+
GGML_UNUSED(model);
|
|
2395
|
+
}
|
|
2396
|
+
|
|
2375
2397
|
static ggml_backend_buffer_type_t llama_default_buffer_type_offload(const llama_model & model, int gpu) {
|
|
2376
2398
|
ggml_backend_buffer_type_t buft = nullptr;
|
|
2377
2399
|
|
|
2378
|
-
#
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2400
|
+
#if defined(GGML_USE_RPC)
|
|
2401
|
+
int dev_count = (int)llama_get_device_count(model);
|
|
2402
|
+
int rpc_count = (int)model.rpc_servers.size();
|
|
2403
|
+
if (gpu >= dev_count - rpc_count) {
|
|
2404
|
+
const char * endpoint = model.rpc_servers[gpu - dev_count + rpc_count].c_str();
|
|
2405
|
+
return ggml_backend_rpc_buffer_type(endpoint);
|
|
2406
|
+
}
|
|
2407
|
+
#endif
|
|
2408
|
+
#if defined(GGML_USE_METAL)
|
|
2382
2409
|
buft = ggml_backend_metal_buffer_type();
|
|
2383
2410
|
#elif defined(GGML_USE_CUDA)
|
|
2384
2411
|
buft = ggml_backend_cuda_buffer_type(gpu);
|
|
@@ -2386,8 +2413,6 @@ static ggml_backend_buffer_type_t llama_default_buffer_type_offload(const llama_
|
|
|
2386
2413
|
buft = ggml_backend_vk_buffer_type(gpu);
|
|
2387
2414
|
#elif defined(GGML_USE_SYCL)
|
|
2388
2415
|
buft = ggml_backend_sycl_buffer_type(gpu);
|
|
2389
|
-
#elif defined(GGML_USE_CLBLAST)
|
|
2390
|
-
buft = ggml_backend_opencl_buffer_type();
|
|
2391
2416
|
#elif defined(GGML_USE_KOMPUTE)
|
|
2392
2417
|
buft = ggml_backend_kompute_buffer_type(gpu);
|
|
2393
2418
|
if (buft == nullptr) {
|
|
@@ -2426,29 +2451,19 @@ static ggml_backend_buffer_type_t llama_default_buffer_type_split(const llama_mo
|
|
|
2426
2451
|
GGML_UNUSED(tensor_split);
|
|
2427
2452
|
}
|
|
2428
2453
|
|
|
2429
|
-
static size_t llama_get_device_count(const llama_model & model) {
|
|
2430
|
-
#if defined(GGML_USE_RPC)
|
|
2431
|
-
return model.rpc_servers.size();
|
|
2432
|
-
#elif defined(GGML_USE_CUDA)
|
|
2433
|
-
return ggml_backend_cuda_get_device_count();
|
|
2434
|
-
#elif defined(GGML_USE_SYCL)
|
|
2435
|
-
return ggml_backend_sycl_get_device_count();
|
|
2436
|
-
#elif defined(GGML_USE_VULKAN)
|
|
2437
|
-
return ggml_backend_vk_get_device_count();
|
|
2438
|
-
#else
|
|
2439
|
-
return 1;
|
|
2440
|
-
#endif
|
|
2441
|
-
GGML_UNUSED(model);
|
|
2442
|
-
}
|
|
2443
|
-
|
|
2444
2454
|
static size_t llama_get_device_memory(const llama_model & model, int device) {
|
|
2445
2455
|
#if defined(GGML_USE_RPC)
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2456
|
+
int dev_count = (int)llama_get_device_count(model);
|
|
2457
|
+
int rpc_count = (int)model.rpc_servers.size();
|
|
2458
|
+
if (device >= dev_count - rpc_count) {
|
|
2459
|
+
size_t total;
|
|
2460
|
+
size_t free;
|
|
2461
|
+
const char * endpoint = model.rpc_servers[device - dev_count + rpc_count].c_str();
|
|
2462
|
+
ggml_backend_rpc_get_device_memory(endpoint, &free, &total);
|
|
2463
|
+
return free;
|
|
2464
|
+
}
|
|
2465
|
+
#endif
|
|
2466
|
+
#if defined(GGML_USE_CUDA)
|
|
2452
2467
|
size_t total;
|
|
2453
2468
|
size_t free;
|
|
2454
2469
|
ggml_backend_cuda_get_device_memory(device, &free, &total);
|
|
@@ -2520,10 +2535,6 @@ static bool llama_kv_cache_init(
|
|
|
2520
2535
|
}
|
|
2521
2536
|
}
|
|
2522
2537
|
|
|
2523
|
-
#ifdef GGML_USE_CLBLAST
|
|
2524
|
-
offload = false;
|
|
2525
|
-
#endif
|
|
2526
|
-
|
|
2527
2538
|
// count used buffer types
|
|
2528
2539
|
std::map<ggml_backend_buffer_type_t, int> buft_layer_count;
|
|
2529
2540
|
if (offload) {
|
|
@@ -4003,8 +4014,8 @@ static void llm_load_hparams(
|
|
|
4003
4014
|
ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);
|
|
4004
4015
|
hparams.rope_finetuned = rope_finetuned;
|
|
4005
4016
|
|
|
4006
|
-
hparams.
|
|
4007
|
-
ml.get_key(LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, hparams.
|
|
4017
|
+
hparams.n_ctx_orig_yarn = hparams.n_ctx_train;
|
|
4018
|
+
ml.get_key(LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, hparams.n_ctx_orig_yarn, false);
|
|
4008
4019
|
|
|
4009
4020
|
// rope_freq_base (optional)
|
|
4010
4021
|
hparams.rope_freq_base_train = 10000.0f;
|
|
@@ -4550,35 +4561,6 @@ static void llm_load_vocab(
|
|
|
4550
4561
|
vocab.special_cls_id = -1;
|
|
4551
4562
|
vocab.special_mask_id = -1;
|
|
4552
4563
|
|
|
4553
|
-
// For Fill-In-the-Middle (FIM)/infill models which where converted
|
|
4554
|
-
// prior to support of FIM special tokens in GGUF, the following
|
|
4555
|
-
// will allow those models to continue to work. The general names
|
|
4556
|
-
// of the known models are currently CodeLlama (LLM_ARCH_LLAMA) and
|
|
4557
|
-
// CodeGemma (LLM_ARCH_GEMMA). This can potentially be removed once
|
|
4558
|
-
// new versions of these models have been published.
|
|
4559
|
-
std::string gen_name;
|
|
4560
|
-
ml.get_key(LLM_KV_GENERAL_NAME, gen_name, false);
|
|
4561
|
-
|
|
4562
|
-
std::transform(gen_name.begin(), gen_name.end(), gen_name.begin(),
|
|
4563
|
-
[](unsigned char c){ return std::tolower(c); });
|
|
4564
|
-
|
|
4565
|
-
if (gen_name.find("code") != std::string::npos) {
|
|
4566
|
-
if (model.arch == LLM_ARCH_LLAMA) {
|
|
4567
|
-
vocab.special_prefix_id = 32007;
|
|
4568
|
-
vocab.special_suffix_id = 32008;
|
|
4569
|
-
vocab.special_middle_id = 32009;
|
|
4570
|
-
vocab.special_eot_id = 32010;
|
|
4571
|
-
} else if (model.arch == LLM_ARCH_GEMMA) {
|
|
4572
|
-
vocab.special_prefix_id = 67;
|
|
4573
|
-
vocab.special_suffix_id = 69;
|
|
4574
|
-
vocab.special_middle_id = 68;
|
|
4575
|
-
// TODO: this is not EOT, it is "file separator" token, needs fix
|
|
4576
|
-
// https://huggingface.co/google/codegemma-7b-it/blob/9b1d9231388358c04d90bd003458f5070d97db44/tokenizer_config.json#L565-L572
|
|
4577
|
-
//vocab.special_eot_id = 70;
|
|
4578
|
-
vocab.special_eot_id = 107;
|
|
4579
|
-
}
|
|
4580
|
-
}
|
|
4581
|
-
|
|
4582
4564
|
const int add_space_prefix_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_ADD_PREFIX).c_str());
|
|
4583
4565
|
if (add_space_prefix_keyidx != -1) {
|
|
4584
4566
|
vocab.add_space_prefix = gguf_get_val_bool(ctx, add_space_prefix_keyidx);
|
|
@@ -4651,8 +4633,7 @@ static void llm_load_vocab(
|
|
|
4651
4633
|
LLAMA_LOG_WARN("%s: ************************************ \n", __func__);
|
|
4652
4634
|
LLAMA_LOG_WARN("%s: \n", __func__);
|
|
4653
4635
|
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
|
|
4654
|
-
} else if (
|
|
4655
|
-
tokenizer_pre == "default") {
|
|
4636
|
+
} else if (tokenizer_pre == "default") {
|
|
4656
4637
|
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
|
|
4657
4638
|
} else if (
|
|
4658
4639
|
tokenizer_pre == "llama3" ||
|
|
@@ -4679,7 +4660,8 @@ static void llm_load_vocab(
|
|
|
4679
4660
|
tokenizer_pre == "jina-es" ||
|
|
4680
4661
|
tokenizer_pre == "jina-de" ||
|
|
4681
4662
|
tokenizer_pre == "jina-v2-es" ||
|
|
4682
|
-
tokenizer_pre == "jina-v2-de"
|
|
4663
|
+
tokenizer_pre == "jina-v2-de" ||
|
|
4664
|
+
tokenizer_pre == "jina-v2-code") {
|
|
4683
4665
|
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2;
|
|
4684
4666
|
} else if (
|
|
4685
4667
|
tokenizer_pre == "refact") {
|
|
@@ -4702,6 +4684,9 @@ static void llm_load_vocab(
|
|
|
4702
4684
|
} else if (
|
|
4703
4685
|
tokenizer_pre == "smaug-bpe") {
|
|
4704
4686
|
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMAUG;
|
|
4687
|
+
} else if (
|
|
4688
|
+
tokenizer_pre == "poro-chat") {
|
|
4689
|
+
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_PORO;
|
|
4705
4690
|
} else {
|
|
4706
4691
|
throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
|
|
4707
4692
|
}
|
|
@@ -4740,12 +4725,64 @@ static void llm_load_vocab(
|
|
|
4740
4725
|
auto & token_data = vocab.id_to_token[i];
|
|
4741
4726
|
token_data.text = std::move(word);
|
|
4742
4727
|
token_data.score = scores ? scores[i] : 0.0f;
|
|
4743
|
-
token_data.
|
|
4728
|
+
token_data.attr = LLAMA_TOKEN_ATTR_NORMAL;
|
|
4729
|
+
|
|
4730
|
+
if (toktypes) { //TODO: remove, required until per token attributes are available from GGUF file
|
|
4731
|
+
switch(toktypes[i]) {
|
|
4732
|
+
case LLAMA_TOKEN_TYPE_UNKNOWN: token_data.attr = LLAMA_TOKEN_ATTR_UNKNOWN; break;
|
|
4733
|
+
case LLAMA_TOKEN_TYPE_UNUSED: token_data.attr = LLAMA_TOKEN_ATTR_UNUSED; break;
|
|
4734
|
+
case LLAMA_TOKEN_TYPE_NORMAL: token_data.attr = LLAMA_TOKEN_ATTR_NORMAL; break;
|
|
4735
|
+
case LLAMA_TOKEN_TYPE_CONTROL: token_data.attr = LLAMA_TOKEN_ATTR_CONTROL; break;
|
|
4736
|
+
case LLAMA_TOKEN_TYPE_USER_DEFINED: token_data.attr = LLAMA_TOKEN_ATTR_USER_DEFINED; break;
|
|
4737
|
+
case LLAMA_TOKEN_TYPE_BYTE: token_data.attr = LLAMA_TOKEN_ATTR_BYTE; break;
|
|
4738
|
+
case LLAMA_TOKEN_TYPE_UNDEFINED: token_data.attr = LLAMA_TOKEN_ATTR_UNDEFINED; break;
|
|
4739
|
+
default: token_data.attr = LLAMA_TOKEN_ATTR_UNDEFINED; break;
|
|
4740
|
+
}
|
|
4741
|
+
}
|
|
4744
4742
|
}
|
|
4745
4743
|
GGML_ASSERT(vocab.id_to_token.size() == vocab.token_to_id.size());
|
|
4746
4744
|
|
|
4747
4745
|
// determine the newline token: LLaMA "<0x0A>" == 10 == '\n', Falcon 193 == '\n'
|
|
4748
4746
|
if (vocab.type == LLAMA_VOCAB_TYPE_SPM) {
|
|
4747
|
+
// For Fill-In-the-Middle (FIM)/infill models which where converted
|
|
4748
|
+
// prior to support of FIM special tokens in GGUF, the following
|
|
4749
|
+
// will allow those models to continue to work. The general names
|
|
4750
|
+
// of the known models are currently CodeLlama (LLM_ARCH_LLAMA) and
|
|
4751
|
+
// CodeGemma (LLM_ARCH_GEMMA). This can potentially be removed once
|
|
4752
|
+
// new versions of these models have been published.
|
|
4753
|
+
std::string gen_name;
|
|
4754
|
+
ml.get_key(LLM_KV_GENERAL_NAME, gen_name, false);
|
|
4755
|
+
|
|
4756
|
+
std::transform(gen_name.begin(), gen_name.end(), gen_name.begin(),
|
|
4757
|
+
[](unsigned char c){ return std::tolower(c); });
|
|
4758
|
+
|
|
4759
|
+
if (gen_name.find("code") != std::string::npos) {
|
|
4760
|
+
if (model.arch == LLM_ARCH_LLAMA
|
|
4761
|
+
&& 32010 < vocab.id_to_token.size()
|
|
4762
|
+
&& vocab.id_to_token[32007].text == "<PRE>"
|
|
4763
|
+
&& vocab.id_to_token[32008].text == "<SUF>"
|
|
4764
|
+
&& vocab.id_to_token[32009].text == "<MID>"
|
|
4765
|
+
&& vocab.id_to_token[32010].text == "<EOT>") {
|
|
4766
|
+
vocab.special_prefix_id = 32007;
|
|
4767
|
+
vocab.special_suffix_id = 32008;
|
|
4768
|
+
vocab.special_middle_id = 32009;
|
|
4769
|
+
vocab.special_eot_id = 32010;
|
|
4770
|
+
} else if (model.arch == LLM_ARCH_GEMMA
|
|
4771
|
+
&& 107 < vocab.id_to_token.size()
|
|
4772
|
+
&& vocab.id_to_token[67].text == "<|fim_prefix|>"
|
|
4773
|
+
&& vocab.id_to_token[69].text == "<|fim_suffix|>"
|
|
4774
|
+
&& vocab.id_to_token[68].text == "<|fim_middle|>"
|
|
4775
|
+
&& vocab.id_to_token[107].text == "<end_of_turn>") {
|
|
4776
|
+
vocab.special_prefix_id = 67;
|
|
4777
|
+
vocab.special_suffix_id = 69;
|
|
4778
|
+
vocab.special_middle_id = 68;
|
|
4779
|
+
// TODO: this is not EOT, it is "file separator" token, needs fix
|
|
4780
|
+
// https://huggingface.co/google/codegemma-7b-it/blob/9b1d9231388358c04d90bd003458f5070d97db44/tokenizer_config.json#L565-L572
|
|
4781
|
+
//vocab.special_eot_id = 70;
|
|
4782
|
+
vocab.special_eot_id = 107;
|
|
4783
|
+
}
|
|
4784
|
+
}
|
|
4785
|
+
|
|
4749
4786
|
try {
|
|
4750
4787
|
vocab.linefeed_id = llama_byte_to_token(vocab, '\n');
|
|
4751
4788
|
} catch (const std::exception & e) {
|
|
@@ -4831,7 +4868,7 @@ static void llm_load_vocab(
|
|
|
4831
4868
|
// build special tokens cache
|
|
4832
4869
|
{
|
|
4833
4870
|
for (llama_vocab::id id = 0; id < (llama_vocab::id)n_vocab; ++id) {
|
|
4834
|
-
if (vocab.id_to_token[id].
|
|
4871
|
+
if (!(vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_NORMAL)) {
|
|
4835
4872
|
vocab.cache_special_tokens.push_back(id);
|
|
4836
4873
|
}
|
|
4837
4874
|
}
|
|
@@ -4845,26 +4882,75 @@ static void llm_load_vocab(
|
|
|
4845
4882
|
LLAMA_LOG_INFO("%s: special tokens cache size = %u\n", __func__, (uint32_t)vocab.cache_special_tokens.size());
|
|
4846
4883
|
}
|
|
4847
4884
|
|
|
4848
|
-
// build token to piece
|
|
4885
|
+
// build token to piece cache
|
|
4849
4886
|
{
|
|
4850
4887
|
size_t size_cache = 0;
|
|
4851
4888
|
|
|
4852
|
-
std::vector<llama_vocab::token> cache_token_to_piece
|
|
4853
|
-
std::vector<llama_vocab::token> cache_token_to_piece_special(n_vocab);
|
|
4889
|
+
std::vector<llama_vocab::token> cache_token_to_piece(n_vocab);
|
|
4854
4890
|
|
|
4855
4891
|
for (uint32_t id = 0; id < n_vocab; ++id) {
|
|
4856
|
-
cache_token_to_piece[id]
|
|
4857
|
-
cache_token_to_piece_special[id] = llama_token_to_piece(&model, id, true);
|
|
4892
|
+
cache_token_to_piece[id] = llama_token_to_piece(&model, id, true);
|
|
4858
4893
|
|
|
4859
4894
|
size_cache += cache_token_to_piece[id].size();
|
|
4860
|
-
size_cache += cache_token_to_piece_special[id].size();
|
|
4861
4895
|
}
|
|
4862
4896
|
|
|
4863
|
-
std::swap(vocab.cache_token_to_piece,
|
|
4864
|
-
std::swap(vocab.cache_token_to_piece_special, cache_token_to_piece_special);
|
|
4897
|
+
std::swap(vocab.cache_token_to_piece, cache_token_to_piece);
|
|
4865
4898
|
|
|
4866
4899
|
LLAMA_LOG_INFO("%s: token to piece cache size = %.4f MB\n", __func__, size_cache / 1024.0 / 1024.0);
|
|
4867
4900
|
}
|
|
4901
|
+
|
|
4902
|
+
// Handle per token attributes
|
|
4903
|
+
//NOTE: Each model customizes per token attributes.
|
|
4904
|
+
//NOTE: Per token attributes are missing from the GGUF file.
|
|
4905
|
+
//TODO: Extract attributes from GGUF file.
|
|
4906
|
+
{
|
|
4907
|
+
auto _contains_any = [] (const std::string &str, const std::vector<std::string> &substrs) -> bool {
|
|
4908
|
+
for (auto substr : substrs) {
|
|
4909
|
+
if (str.find(substr) < std::string::npos) {
|
|
4910
|
+
return true;
|
|
4911
|
+
}
|
|
4912
|
+
}
|
|
4913
|
+
return false;
|
|
4914
|
+
};
|
|
4915
|
+
|
|
4916
|
+
auto _set_tokenid_attr = [&] (const llama_vocab::id id, llama_token_attr attr, bool value) {
|
|
4917
|
+
uint32_t current = vocab.id_to_token.at(id).attr;
|
|
4918
|
+
current = value ? (current | attr) : (current & ~attr);
|
|
4919
|
+
vocab.id_to_token[id].attr = (llama_token_attr) current;
|
|
4920
|
+
};
|
|
4921
|
+
|
|
4922
|
+
auto _set_token_attr = [&] (const std::string & token, llama_token_attr attr, bool value) {
|
|
4923
|
+
_set_tokenid_attr(vocab.token_to_id.at(token), attr, value);
|
|
4924
|
+
};
|
|
4925
|
+
|
|
4926
|
+
std::string model_name;
|
|
4927
|
+
std::string tokenizer_pre;
|
|
4928
|
+
|
|
4929
|
+
ml.get_key(LLM_KV_GENERAL_NAME, model_name, false);
|
|
4930
|
+
ml.get_key(LLM_KV_TOKENIZER_PRE, tokenizer_pre, false);
|
|
4931
|
+
|
|
4932
|
+
// model name to lowercase
|
|
4933
|
+
std::transform(model_name.begin(), model_name.end(), model_name.begin(),
|
|
4934
|
+
[] (const std::string::value_type x) {
|
|
4935
|
+
return std::tolower(x);
|
|
4936
|
+
}
|
|
4937
|
+
);
|
|
4938
|
+
|
|
4939
|
+
// set attributes by model/tokenizer name
|
|
4940
|
+
if (_contains_any(tokenizer_pre, {"jina-v2-es", "jina-v2-de"})) {
|
|
4941
|
+
_set_token_attr("<mask>", LLAMA_TOKEN_ATTR_LSTRIP, true);
|
|
4942
|
+
} else if (_contains_any(model_name, {"phi-3", "phi3"})) {
|
|
4943
|
+
for (auto id : vocab.cache_special_tokens) {
|
|
4944
|
+
_set_tokenid_attr(id, LLAMA_TOKEN_ATTR_RSTRIP, true);
|
|
4945
|
+
}
|
|
4946
|
+
for (auto token : {"</s>"}) {
|
|
4947
|
+
_set_token_attr(token, LLAMA_TOKEN_ATTR_RSTRIP, true);
|
|
4948
|
+
}
|
|
4949
|
+
for (auto token : {"<unk>", "<s>", "<|endoftext|>"}) {
|
|
4950
|
+
_set_token_attr(token, LLAMA_TOKEN_ATTR_RSTRIP, false);
|
|
4951
|
+
}
|
|
4952
|
+
}
|
|
4953
|
+
}
|
|
4868
4954
|
}
|
|
4869
4955
|
|
|
4870
4956
|
static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {
|
|
@@ -4904,7 +4990,7 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {
|
|
|
4904
4990
|
LLAMA_LOG_INFO("%s: rope scaling = %s\n", __func__, rope_scaling_type);
|
|
4905
4991
|
LLAMA_LOG_INFO("%s: freq_base_train = %.1f\n", __func__, hparams.rope_freq_base_train);
|
|
4906
4992
|
LLAMA_LOG_INFO("%s: freq_scale_train = %g\n", __func__, hparams.rope_freq_scale_train);
|
|
4907
|
-
LLAMA_LOG_INFO("%s:
|
|
4993
|
+
LLAMA_LOG_INFO("%s: n_ctx_orig_yarn = %u\n", __func__, hparams.n_ctx_orig_yarn);
|
|
4908
4994
|
LLAMA_LOG_INFO("%s: rope_finetuned = %s\n", __func__, hparams.rope_finetuned ? "yes" : "unknown");
|
|
4909
4995
|
LLAMA_LOG_INFO("%s: ssm_d_conv = %u\n", __func__, hparams.ssm_d_conv);
|
|
4910
4996
|
LLAMA_LOG_INFO("%s: ssm_d_inner = %u\n", __func__, hparams.ssm_d_inner);
|
|
@@ -5129,12 +5215,10 @@ static bool llm_load_tensors(
|
|
|
5129
5215
|
// output
|
|
5130
5216
|
{
|
|
5131
5217
|
model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
|
|
5132
|
-
|
|
5133
|
-
|
|
5134
|
-
|
|
5135
|
-
|
|
5136
|
-
model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);
|
|
5137
|
-
}
|
|
5218
|
+
model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
|
|
5219
|
+
// if output is NULL, init from the input tok embed
|
|
5220
|
+
if (model.output == NULL) {
|
|
5221
|
+
model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);
|
|
5138
5222
|
}
|
|
5139
5223
|
}
|
|
5140
5224
|
|
|
@@ -5453,7 +5537,7 @@ static bool llm_load_tensors(
|
|
|
5453
5537
|
|
|
5454
5538
|
layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});
|
|
5455
5539
|
} else {
|
|
5456
|
-
layer.ffn_gate
|
|
5540
|
+
layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
|
|
5457
5541
|
}
|
|
5458
5542
|
|
|
5459
5543
|
layer.layer_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd});
|
|
@@ -5494,6 +5578,9 @@ static bool llm_load_tensors(
|
|
|
5494
5578
|
layer.attn_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}); //output_norm
|
|
5495
5579
|
layer.attn_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd});
|
|
5496
5580
|
|
|
5581
|
+
layer.attn_norm_2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
|
|
5582
|
+
layer.attn_norm_2_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
|
|
5583
|
+
|
|
5497
5584
|
layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
|
|
5498
5585
|
layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
|
|
5499
5586
|
|
|
@@ -7072,7 +7159,7 @@ struct llm_build_context {
|
|
|
7072
7159
|
const int32_t n_kv; // size of KV cache to consider (n_kv <= kv_self.size)
|
|
7073
7160
|
const int32_t n_outputs;
|
|
7074
7161
|
const int32_t kv_head; // index of where we store new KV data in the cache
|
|
7075
|
-
const int32_t
|
|
7162
|
+
const int32_t n_ctx_orig;
|
|
7076
7163
|
|
|
7077
7164
|
const bool flash_attn;
|
|
7078
7165
|
|
|
@@ -7121,7 +7208,7 @@ struct llm_build_context {
|
|
|
7121
7208
|
n_kv (worst_case ? kv_self.size : kv_self.n),
|
|
7122
7209
|
n_outputs (worst_case ? n_tokens : lctx.n_outputs),
|
|
7123
7210
|
kv_head (worst_case ? (kv_self.recurrent ? 0 : kv_self.size - n_tokens) : kv_self.head),
|
|
7124
|
-
|
|
7211
|
+
n_ctx_orig (cparams.n_ctx_orig_yarn),
|
|
7125
7212
|
flash_attn (cparams.flash_attn),
|
|
7126
7213
|
pooling_type (cparams.pooling_type),
|
|
7127
7214
|
rope_type (hparams.rope_type),
|
|
@@ -7179,7 +7266,7 @@ struct llm_build_context {
|
|
|
7179
7266
|
ggml_row_size(kv_self.k_l[il]->type, n_embd_head_k),
|
|
7180
7267
|
ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa),
|
|
7181
7268
|
0),
|
|
7182
|
-
lctx.inp_K_shift, rope_factors, n_rot, rope_type,
|
|
7269
|
+
lctx.inp_K_shift, rope_factors, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
7183
7270
|
ext_factor, attn_factor, beta_fast, beta_slow);
|
|
7184
7271
|
|
|
7185
7272
|
cb(tmp, "K_shifted", il);
|
|
@@ -7288,7 +7375,7 @@ struct llm_build_context {
|
|
|
7288
7375
|
// choose long/short freq factors based on the context size
|
|
7289
7376
|
const auto n_ctx_pre_seq = cparams.n_ctx / cparams.n_seq_max;
|
|
7290
7377
|
|
|
7291
|
-
if (n_ctx_pre_seq > hparams.
|
|
7378
|
+
if (n_ctx_pre_seq > hparams.n_ctx_orig_yarn) {
|
|
7292
7379
|
return model.layers[il].rope_long;
|
|
7293
7380
|
}
|
|
7294
7381
|
|
|
@@ -7404,14 +7491,14 @@ struct llm_build_context {
|
|
|
7404
7491
|
|
|
7405
7492
|
Qcur = ggml_rope_ext(
|
|
7406
7493
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
7407
|
-
n_rot, rope_type,
|
|
7494
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
7408
7495
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
7409
7496
|
);
|
|
7410
7497
|
cb(Qcur, "Qcur", il);
|
|
7411
7498
|
|
|
7412
7499
|
Kcur = ggml_rope_ext(
|
|
7413
7500
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
7414
|
-
n_rot, rope_type,
|
|
7501
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
7415
7502
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
7416
7503
|
);
|
|
7417
7504
|
cb(Kcur, "Kcur", il);
|
|
@@ -7535,12 +7622,12 @@ struct llm_build_context {
|
|
|
7535
7622
|
case MODEL_7B:
|
|
7536
7623
|
Qcur = ggml_rope_ext(
|
|
7537
7624
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
7538
|
-
n_rot, rope_type,
|
|
7625
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
7539
7626
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
7540
7627
|
);
|
|
7541
7628
|
Kcur = ggml_rope_ext(
|
|
7542
7629
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
7543
|
-
n_rot, rope_type,
|
|
7630
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
7544
7631
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
7545
7632
|
);
|
|
7546
7633
|
break;
|
|
@@ -7647,14 +7734,14 @@ struct llm_build_context {
|
|
|
7647
7734
|
|
|
7648
7735
|
Qcur = ggml_rope_ext(
|
|
7649
7736
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
7650
|
-
n_rot, rope_type,
|
|
7737
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
7651
7738
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
7652
7739
|
);
|
|
7653
7740
|
cb(Qcur, "Qcur", il);
|
|
7654
7741
|
|
|
7655
7742
|
Kcur = ggml_rope_ext(
|
|
7656
7743
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
7657
|
-
n_rot, rope_type,
|
|
7744
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
7658
7745
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
7659
7746
|
);
|
|
7660
7747
|
cb(Kcur, "Kcur", il);
|
|
@@ -7767,13 +7854,13 @@ struct llm_build_context {
|
|
|
7767
7854
|
|
|
7768
7855
|
// using mode = 2 for neox mode
|
|
7769
7856
|
Qcur = ggml_rope_ext(
|
|
7770
|
-
ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type,
|
|
7857
|
+
ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
|
|
7771
7858
|
freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
|
|
7772
7859
|
);
|
|
7773
7860
|
cb(Qcur, "Qcur", il);
|
|
7774
7861
|
|
|
7775
7862
|
Kcur = ggml_rope_ext(
|
|
7776
|
-
ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type,
|
|
7863
|
+
ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
|
|
7777
7864
|
freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
|
|
7778
7865
|
);
|
|
7779
7866
|
cb(Kcur, "Kcur", il);
|
|
@@ -7891,14 +7978,14 @@ struct llm_build_context {
|
|
|
7891
7978
|
|
|
7892
7979
|
Qcur = ggml_rope_ext(
|
|
7893
7980
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
7894
|
-
n_rot, rope_type,
|
|
7981
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
7895
7982
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
7896
7983
|
);
|
|
7897
7984
|
cb(Qcur, "Qcur", il);
|
|
7898
7985
|
|
|
7899
7986
|
Kcur = ggml_rope_ext(
|
|
7900
7987
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
7901
|
-
n_rot, rope_type,
|
|
7988
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
7902
7989
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
7903
7990
|
);
|
|
7904
7991
|
cb(Kcur, "Kcur", il);
|
|
@@ -8044,14 +8131,14 @@ struct llm_build_context {
|
|
|
8044
8131
|
|
|
8045
8132
|
Qcur = ggml_rope_ext(
|
|
8046
8133
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
8047
|
-
n_rot, rope_type,
|
|
8134
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
8048
8135
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
8049
8136
|
);
|
|
8050
8137
|
cb(Qcur, "Qcur", il);
|
|
8051
8138
|
|
|
8052
8139
|
Kcur = ggml_rope_ext(
|
|
8053
8140
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
8054
|
-
n_rot, rope_type,
|
|
8141
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
8055
8142
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
8056
8143
|
);
|
|
8057
8144
|
cb(Kcur, "Kcur", il);
|
|
@@ -8398,14 +8485,14 @@ struct llm_build_context {
|
|
|
8398
8485
|
|
|
8399
8486
|
Qcur = ggml_rope_ext(
|
|
8400
8487
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
8401
|
-
n_rot, rope_type,
|
|
8488
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
8402
8489
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
8403
8490
|
);
|
|
8404
8491
|
cb(Qcur, "Qcur", il);
|
|
8405
8492
|
|
|
8406
8493
|
Kcur = ggml_rope_ext(
|
|
8407
8494
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
8408
|
-
n_rot, rope_type,
|
|
8495
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
8409
8496
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
8410
8497
|
);
|
|
8411
8498
|
cb(Kcur, "Kcur", il);
|
|
@@ -8457,6 +8544,11 @@ struct llm_build_context {
|
|
|
8457
8544
|
// attention layer norm
|
|
8458
8545
|
cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_out_norm, model.layers[il].attn_out_norm_b, LLM_NORM, cb, il);
|
|
8459
8546
|
|
|
8547
|
+
if (model.layers[il].attn_norm_2 != nullptr) {
|
|
8548
|
+
cur = ggml_add(ctx0, cur, inpL); // re-add the layer input
|
|
8549
|
+
cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_norm_2, model.layers[il].attn_norm_2_b, LLM_NORM, cb, il);
|
|
8550
|
+
}
|
|
8551
|
+
|
|
8460
8552
|
struct ggml_tensor * ffn_inp = cur;
|
|
8461
8553
|
cb(ffn_inp, "ffn_inp", il);
|
|
8462
8554
|
|
|
@@ -8838,14 +8930,14 @@ struct llm_build_context {
|
|
|
8838
8930
|
|
|
8839
8931
|
Qcur = ggml_rope_ext(
|
|
8840
8932
|
ctx0, Qcur, inp_pos, nullptr,
|
|
8841
|
-
n_rot, rope_type,
|
|
8933
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
8842
8934
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
8843
8935
|
);
|
|
8844
8936
|
cb(Qcur, "Qcur", il);
|
|
8845
8937
|
|
|
8846
8938
|
Kcur = ggml_rope_ext(
|
|
8847
8939
|
ctx0, Kcur, inp_pos, nullptr,
|
|
8848
|
-
n_rot, rope_type,
|
|
8940
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
8849
8941
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
8850
8942
|
);
|
|
8851
8943
|
cb(Kcur, "Kcur", il);
|
|
@@ -8957,13 +9049,13 @@ struct llm_build_context {
|
|
|
8957
9049
|
|
|
8958
9050
|
// using mode = 2 for neox mode
|
|
8959
9051
|
Qcur = ggml_rope_ext(
|
|
8960
|
-
ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type,
|
|
9052
|
+
ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
|
|
8961
9053
|
freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
|
|
8962
9054
|
);
|
|
8963
9055
|
cb(Qcur, "Qcur", il);
|
|
8964
9056
|
|
|
8965
9057
|
Kcur = ggml_rope_ext(
|
|
8966
|
-
ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type,
|
|
9058
|
+
ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
|
|
8967
9059
|
freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
|
|
8968
9060
|
);
|
|
8969
9061
|
cb(Kcur, "Kcur", il);
|
|
@@ -9069,14 +9161,14 @@ struct llm_build_context {
|
|
|
9069
9161
|
|
|
9070
9162
|
Qcur = ggml_rope_ext(
|
|
9071
9163
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
9072
|
-
n_rot, rope_type,
|
|
9164
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
9073
9165
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
9074
9166
|
);
|
|
9075
9167
|
cb(Qcur, "Qcur", il);
|
|
9076
9168
|
|
|
9077
9169
|
Kcur = ggml_rope_ext(
|
|
9078
9170
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
9079
|
-
n_rot, rope_type,
|
|
9171
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
9080
9172
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
9081
9173
|
);
|
|
9082
9174
|
cb(Kcur, "Kcur", il);
|
|
@@ -9183,14 +9275,14 @@ struct llm_build_context {
|
|
|
9183
9275
|
|
|
9184
9276
|
Qcur = ggml_rope_ext(
|
|
9185
9277
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
9186
|
-
n_rot, rope_type,
|
|
9278
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
9187
9279
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
9188
9280
|
);
|
|
9189
9281
|
cb(Qcur, "Qcur", il);
|
|
9190
9282
|
|
|
9191
9283
|
Kcur = ggml_rope_ext(
|
|
9192
9284
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
9193
|
-
n_rot, rope_type,
|
|
9285
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
9194
9286
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
9195
9287
|
);
|
|
9196
9288
|
cb(Kcur, "Kcur", il);
|
|
@@ -9335,7 +9427,7 @@ struct llm_build_context {
|
|
|
9335
9427
|
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
|
|
9336
9428
|
|
|
9337
9429
|
Qcur = ggml_rope_ext(
|
|
9338
|
-
ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type,
|
|
9430
|
+
ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
|
|
9339
9431
|
freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
|
|
9340
9432
|
);
|
|
9341
9433
|
cb(Qcur, "Qcur", il);
|
|
@@ -9346,7 +9438,7 @@ struct llm_build_context {
|
|
|
9346
9438
|
cb(Qcur, "Qcur", il);
|
|
9347
9439
|
|
|
9348
9440
|
Kcur = ggml_rope_ext(
|
|
9349
|
-
ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type,
|
|
9441
|
+
ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
|
|
9350
9442
|
freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
|
|
9351
9443
|
);
|
|
9352
9444
|
cb(Kcur, "Kcur", il);
|
|
@@ -9457,7 +9549,7 @@ struct llm_build_context {
|
|
|
9457
9549
|
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
|
|
9458
9550
|
|
|
9459
9551
|
Qcur = ggml_rope_ext(
|
|
9460
|
-
ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type,
|
|
9552
|
+
ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig,
|
|
9461
9553
|
freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
|
|
9462
9554
|
);
|
|
9463
9555
|
cb(Qcur, "Qcur", il);
|
|
@@ -9466,7 +9558,7 @@ struct llm_build_context {
|
|
|
9466
9558
|
cb(Qcur, "Qcur", il);
|
|
9467
9559
|
|
|
9468
9560
|
Kcur = ggml_rope_ext(
|
|
9469
|
-
ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type,
|
|
9561
|
+
ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, n_ctx_orig,
|
|
9470
9562
|
freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow
|
|
9471
9563
|
);
|
|
9472
9564
|
cb(Kcur, "Kcur", il);
|
|
@@ -9574,13 +9666,13 @@ struct llm_build_context {
|
|
|
9574
9666
|
|
|
9575
9667
|
Qcur = ggml_rope_ext(
|
|
9576
9668
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_rot, n_head, n_tokens), inp_pos, nullptr,
|
|
9577
|
-
n_embd_head, rope_type,
|
|
9669
|
+
n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
9578
9670
|
ext_factor, attn_factor, beta_fast, beta_slow);
|
|
9579
9671
|
cb(Qcur, "Qcur", il);
|
|
9580
9672
|
|
|
9581
9673
|
Kcur = ggml_rope_ext(
|
|
9582
9674
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_rot, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
9583
|
-
n_embd_head, rope_type,
|
|
9675
|
+
n_embd_head, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
9584
9676
|
ext_factor, attn_factor, beta_fast, beta_slow);
|
|
9585
9677
|
cb(Kcur, "Kcur", il);
|
|
9586
9678
|
|
|
@@ -9782,14 +9874,14 @@ struct llm_build_context {
|
|
|
9782
9874
|
|
|
9783
9875
|
struct ggml_tensor * Qcur = ggml_rope_ext(
|
|
9784
9876
|
ctx0, ggml_reshape_3d(ctx0, tmpq, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
9785
|
-
n_rot, rope_type,
|
|
9877
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
9786
9878
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
9787
9879
|
);
|
|
9788
9880
|
cb(Qcur, "Qcur", il);
|
|
9789
9881
|
|
|
9790
9882
|
struct ggml_tensor * Kcur = ggml_rope_ext(
|
|
9791
9883
|
ctx0, ggml_reshape_3d(ctx0, tmpk, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
9792
|
-
n_rot, rope_type,
|
|
9884
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
9793
9885
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
9794
9886
|
);
|
|
9795
9887
|
cb(Kcur, "Kcur", il);
|
|
@@ -9898,14 +9990,14 @@ struct llm_build_context {
|
|
|
9898
9990
|
|
|
9899
9991
|
Qcur = ggml_rope_ext(
|
|
9900
9992
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
9901
|
-
n_rot, rope_type,
|
|
9993
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
9902
9994
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
9903
9995
|
);
|
|
9904
9996
|
cb(Qcur, "Qcur", il);
|
|
9905
9997
|
|
|
9906
9998
|
Kcur = ggml_rope_ext(
|
|
9907
9999
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
9908
|
-
n_rot, rope_type,
|
|
10000
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
9909
10001
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
9910
10002
|
);
|
|
9911
10003
|
cb(Kcur, "Kcur", il);
|
|
@@ -10015,14 +10107,14 @@ struct llm_build_context {
|
|
|
10015
10107
|
|
|
10016
10108
|
Qcur = ggml_rope_ext(
|
|
10017
10109
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
10018
|
-
n_rot, rope_type,
|
|
10110
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
10019
10111
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
10020
10112
|
);
|
|
10021
10113
|
cb(Qcur, "Qcur", il);
|
|
10022
10114
|
|
|
10023
10115
|
Kcur = ggml_rope_ext(
|
|
10024
10116
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
10025
|
-
n_rot, rope_type,
|
|
10117
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
10026
10118
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
10027
10119
|
);
|
|
10028
10120
|
cb(Kcur, "Kcur", il);
|
|
@@ -10145,14 +10237,14 @@ struct llm_build_context {
|
|
|
10145
10237
|
|
|
10146
10238
|
Qcur = ggml_rope_ext(
|
|
10147
10239
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
10148
|
-
n_rot, rope_type,
|
|
10240
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
10149
10241
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
10150
10242
|
);
|
|
10151
10243
|
cb(Qcur, "Qcur", il);
|
|
10152
10244
|
|
|
10153
10245
|
Kcur = ggml_rope_ext(
|
|
10154
10246
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
10155
|
-
n_rot, rope_type,
|
|
10247
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
10156
10248
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
10157
10249
|
);
|
|
10158
10250
|
cb(Kcur, "Kcur", il);
|
|
@@ -10217,7 +10309,7 @@ struct llm_build_context {
|
|
|
10217
10309
|
cb(cur, "lmhead_scaling", -1);
|
|
10218
10310
|
|
|
10219
10311
|
// lm_head
|
|
10220
|
-
cur = ggml_mul_mat(ctx0, model.
|
|
10312
|
+
cur = ggml_mul_mat(ctx0, model.output, cur);
|
|
10221
10313
|
cb(cur, "result_output", -1);
|
|
10222
10314
|
|
|
10223
10315
|
ggml_build_forward_expand(gf, cur);
|
|
@@ -10265,7 +10357,7 @@ struct llm_build_context {
|
|
|
10265
10357
|
|
|
10266
10358
|
Qcur = ggml_rope_ext(
|
|
10267
10359
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos, nullptr,
|
|
10268
|
-
n_embd_head_k, rope_type,
|
|
10360
|
+
n_embd_head_k, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
10269
10361
|
ext_factor, attn_factor, beta_fast, beta_slow);
|
|
10270
10362
|
cb(Qcur, "Qcur", il);
|
|
10271
10363
|
|
|
@@ -10274,7 +10366,7 @@ struct llm_build_context {
|
|
|
10274
10366
|
|
|
10275
10367
|
Kcur = ggml_rope_ext(
|
|
10276
10368
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
10277
|
-
n_embd_head_k, rope_type,
|
|
10369
|
+
n_embd_head_k, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
10278
10370
|
ext_factor, attn_factor, beta_fast, beta_slow);
|
|
10279
10371
|
cb(Kcur, "Kcur", il);
|
|
10280
10372
|
|
|
@@ -10385,14 +10477,14 @@ struct llm_build_context {
|
|
|
10385
10477
|
|
|
10386
10478
|
Qcur = ggml_rope_ext(
|
|
10387
10479
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
10388
|
-
n_rot, rope_type,
|
|
10480
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
10389
10481
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
10390
10482
|
);
|
|
10391
10483
|
cb(Qcur, "Qcur", il);
|
|
10392
10484
|
|
|
10393
10485
|
Kcur = ggml_rope_ext(
|
|
10394
10486
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
10395
|
-
n_rot, rope_type,
|
|
10487
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
10396
10488
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
10397
10489
|
);
|
|
10398
10490
|
cb(Kcur, "Kcur", il);
|
|
@@ -10675,14 +10767,14 @@ struct llm_build_context {
|
|
|
10675
10767
|
|
|
10676
10768
|
Qcur = ggml_rope_ext(
|
|
10677
10769
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
10678
|
-
n_rot, rope_type,
|
|
10770
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
10679
10771
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
10680
10772
|
);
|
|
10681
10773
|
cb(Qcur, "Qcur", il);
|
|
10682
10774
|
|
|
10683
10775
|
Kcur = ggml_rope_ext(
|
|
10684
10776
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
10685
|
-
n_rot, rope_type,
|
|
10777
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
10686
10778
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
10687
10779
|
);
|
|
10688
10780
|
cb(Kcur, "Kcur", il);
|
|
@@ -10806,14 +10898,14 @@ struct llm_build_context {
|
|
|
10806
10898
|
|
|
10807
10899
|
Qcur = ggml_rope_ext(
|
|
10808
10900
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
10809
|
-
n_rot, rope_type,
|
|
10901
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
10810
10902
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
10811
10903
|
);
|
|
10812
10904
|
cb(Qcur, "Qcur", il);
|
|
10813
10905
|
|
|
10814
10906
|
Kcur = ggml_rope_ext(
|
|
10815
10907
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
10816
|
-
n_rot, rope_type,
|
|
10908
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
10817
10909
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
10818
10910
|
);
|
|
10819
10911
|
cb(Kcur, "Kcur", il);
|
|
@@ -10920,14 +11012,14 @@ struct llm_build_context {
|
|
|
10920
11012
|
|
|
10921
11013
|
Qcur = ggml_rope_ext(
|
|
10922
11014
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
10923
|
-
n_rot, rope_type,
|
|
11015
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
10924
11016
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
10925
11017
|
);
|
|
10926
11018
|
cb(Qcur, "Qcur", il);
|
|
10927
11019
|
|
|
10928
11020
|
Kcur = ggml_rope_ext(
|
|
10929
11021
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
10930
|
-
n_rot, rope_type,
|
|
11022
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
10931
11023
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
10932
11024
|
);
|
|
10933
11025
|
cb(Kcur, "Kcur", il);
|
|
@@ -11055,14 +11147,14 @@ struct llm_build_context {
|
|
|
11055
11147
|
|
|
11056
11148
|
Qcur = ggml_rope_ext(
|
|
11057
11149
|
ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
|
|
11058
|
-
n_rot, rope_type,
|
|
11150
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
11059
11151
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
11060
11152
|
);
|
|
11061
11153
|
cb(Qcur, "Qcur", il);
|
|
11062
11154
|
|
|
11063
11155
|
Kcur = ggml_rope_ext(
|
|
11064
11156
|
ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
|
|
11065
|
-
n_rot, rope_type,
|
|
11157
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
11066
11158
|
ext_factor, attn_factor, beta_fast, beta_slow
|
|
11067
11159
|
);
|
|
11068
11160
|
cb(Kcur, "Kcur", il);
|
|
@@ -11272,7 +11364,7 @@ struct llm_build_context {
|
|
|
11272
11364
|
q_pe = ggml_cont(ctx0, q_pe); // TODO: the CUDA backend does not support non-contiguous RoPE
|
|
11273
11365
|
q_pe = ggml_rope_ext(
|
|
11274
11366
|
ctx0, q_pe, inp_pos, nullptr,
|
|
11275
|
-
n_rot, rope_type,
|
|
11367
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
11276
11368
|
ext_factor, attn_factor_scaled, beta_fast, beta_slow
|
|
11277
11369
|
);
|
|
11278
11370
|
cb(q_pe, "q_pe", il);
|
|
@@ -11281,7 +11373,7 @@ struct llm_build_context {
|
|
|
11281
11373
|
k_pe = ggml_cont(ctx0, k_pe); // TODO: the CUDA backend does not support non-contiguous RoPE
|
|
11282
11374
|
k_pe = ggml_rope_ext(
|
|
11283
11375
|
ctx0, k_pe, inp_pos, nullptr,
|
|
11284
|
-
n_rot, rope_type,
|
|
11376
|
+
n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
|
|
11285
11377
|
ext_factor, attn_factor_scaled, beta_fast, beta_slow
|
|
11286
11378
|
);
|
|
11287
11379
|
cb(k_pe, "k_pe", il);
|
|
@@ -11458,7 +11550,8 @@ static struct ggml_cgraph * llama_build_graph(
|
|
|
11458
11550
|
if (batch.n_tokens < 32 || full_offload) {
|
|
11459
11551
|
if (il != -1 && strcmp(name, "norm") == 0) {
|
|
11460
11552
|
for (auto * backend : lctx.backends) {
|
|
11461
|
-
if (
|
|
11553
|
+
if (ggml_backend_supports_buft(backend, lctx.model.buft_layer[il].buft) &&
|
|
11554
|
+
(ggml_backend_supports_op(backend, cur) || ggml_backend_offload_op(backend, cur))) {
|
|
11462
11555
|
ggml_backend_sched_set_tensor_backend(lctx.sched, cur, backend);
|
|
11463
11556
|
break;
|
|
11464
11557
|
}
|
|
@@ -11955,6 +12048,11 @@ static void llama_graph_compute(
|
|
|
11955
12048
|
ggml_backend_cpu_set_n_threads(lctx.backend_cpu, n_threads);
|
|
11956
12049
|
ggml_backend_cpu_set_abort_callback(lctx.backend_cpu, lctx.abort_callback, lctx.abort_callback_data);
|
|
11957
12050
|
}
|
|
12051
|
+
#ifdef GGML_USE_BLAS
|
|
12052
|
+
if (lctx.backend_blas != nullptr) {
|
|
12053
|
+
ggml_backend_blas_set_n_threads(lctx.backend_blas, n_threads);
|
|
12054
|
+
}
|
|
12055
|
+
#endif
|
|
11958
12056
|
|
|
11959
12057
|
ggml_backend_sched_graph_compute_async(lctx.sched, gf);
|
|
11960
12058
|
|
|
@@ -12177,17 +12275,6 @@ static int llama_decode_internal(
|
|
|
12177
12275
|
}
|
|
12178
12276
|
// LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs);
|
|
12179
12277
|
|
|
12180
|
-
// for big prompts, if BLAS is enabled, it is better to use only one thread
|
|
12181
|
-
// otherwise, the threads are spin-lock waiting for the BLAS calls and are degrading the performance
|
|
12182
|
-
// TODO: this is mostly important for Apple Silicon where CBLAS is still performing very well
|
|
12183
|
-
// we still need some threads to process all non-mul_mat ops, but not too much to avoid interfering
|
|
12184
|
-
// with the BLAS calls. need a better solution
|
|
12185
|
-
// MoE Special Case: This logic applies when hparams.n_expert == 0, i.e. the model is NOT an MoE model. When an MoE is
|
|
12186
|
-
// being processed then Accelerate/BLAS will not be involved, so capping would limit performance.
|
|
12187
|
-
if (n_tokens >= 32 && hparams.n_expert == 0 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas()) {
|
|
12188
|
-
n_threads = std::min(4, n_threads);
|
|
12189
|
-
}
|
|
12190
|
-
|
|
12191
12278
|
ggml_backend_sched_alloc_graph(lctx.sched, gf);
|
|
12192
12279
|
|
|
12193
12280
|
llama_set_inputs(lctx, u_batch);
|
|
@@ -12616,27 +12703,27 @@ static enum llama_vocab_type llama_vocab_get_type(const llama_vocab & vocab) {
|
|
|
12616
12703
|
|
|
12617
12704
|
static bool llama_is_normal_token(const llama_vocab & vocab, llama_token id) {
|
|
12618
12705
|
GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
|
|
12619
|
-
return vocab.id_to_token[id].
|
|
12706
|
+
return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_NORMAL;
|
|
12620
12707
|
}
|
|
12621
12708
|
|
|
12622
12709
|
static bool llama_is_unknown_token(const llama_vocab & vocab, llama_token id) {
|
|
12623
12710
|
GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
|
|
12624
|
-
return vocab.id_to_token[id].
|
|
12711
|
+
return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_UNKNOWN;
|
|
12625
12712
|
}
|
|
12626
12713
|
|
|
12627
12714
|
static bool llama_is_control_token(const llama_vocab & vocab, llama_token id) {
|
|
12628
12715
|
GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
|
|
12629
|
-
return vocab.id_to_token[id].
|
|
12716
|
+
return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_CONTROL;
|
|
12630
12717
|
}
|
|
12631
12718
|
|
|
12632
12719
|
static bool llama_is_byte_token(const llama_vocab & vocab, llama_token id) {
|
|
12633
12720
|
GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
|
|
12634
|
-
return vocab.id_to_token[id].
|
|
12721
|
+
return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_BYTE;
|
|
12635
12722
|
}
|
|
12636
12723
|
|
|
12637
12724
|
static bool llama_is_user_defined_token(const llama_vocab& vocab, llama_token id) {
|
|
12638
12725
|
GGML_ASSERT(vocab.type != LLAMA_VOCAB_TYPE_NONE);
|
|
12639
|
-
return vocab.id_to_token[id].
|
|
12726
|
+
return vocab.id_to_token[id].attr & LLAMA_TOKEN_ATTR_USER_DEFINED;
|
|
12640
12727
|
}
|
|
12641
12728
|
|
|
12642
12729
|
static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) {
|
|
@@ -12954,6 +13041,11 @@ struct llm_tokenizer_bpe {
|
|
|
12954
13041
|
"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
|
|
12955
13042
|
});
|
|
12956
13043
|
break;
|
|
13044
|
+
case LLAMA_VOCAB_PRE_TYPE_PORO:
|
|
13045
|
+
word_collection = unicode_regex_split(text, {
|
|
13046
|
+
" ?[^(\\s|.,!?…。，、।۔،)]+",
|
|
13047
|
+
});
|
|
13048
|
+
break;
|
|
12957
13049
|
default:
|
|
12958
13050
|
// default regex for BPE tokenization pre-processing
|
|
12959
13051
|
word_collection = unicode_regex_split(text, {
|
|
@@ -13254,7 +13346,8 @@ struct fragment_buffer_variant {
|
|
|
13254
13346
|
static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<fragment_buffer_variant> & buffer) {
|
|
13255
13347
|
// for each special token
|
|
13256
13348
|
for (const llama_vocab::id special_id : vocab.cache_special_tokens) {
|
|
13257
|
-
const auto &
|
|
13349
|
+
const auto & data = vocab.id_to_token[special_id];
|
|
13350
|
+
const auto & special_token = data.text;
|
|
13258
13351
|
|
|
13259
13352
|
// for each text fragment
|
|
13260
13353
|
std::forward_list<fragment_buffer_variant>::iterator it = buffer.begin();
|
|
@@ -13291,13 +13384,22 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<
|
|
|
13291
13384
|
if (match > raw_text_base_offset) {
|
|
13292
13385
|
// left
|
|
13293
13386
|
const int64_t left_reminder_offset = raw_text_base_offset + 0;
|
|
13294
|
-
|
|
13295
|
-
|
|
13387
|
+
int64_t left_reminder_length = match - raw_text_base_offset;
|
|
13388
|
+
|
|
13389
|
+
if (data.attr & LLAMA_TOKEN_ATTR_LSTRIP) {
|
|
13390
|
+
while (left_reminder_length > 0 && isspace(raw_text[left_reminder_offset + left_reminder_length - 1])) {
|
|
13391
|
+
left_reminder_length--;
|
|
13392
|
+
}
|
|
13393
|
+
}
|
|
13394
|
+
|
|
13395
|
+
if (left_reminder_length > 0) {
|
|
13396
|
+
buffer.emplace_after(it, raw_text, left_reminder_offset, left_reminder_length);
|
|
13397
|
+
it++;
|
|
13398
|
+
}
|
|
13296
13399
|
|
|
13297
13400
|
#ifdef PRETOKENIZERDEBUG
|
|
13298
13401
|
LLAMA_LOG_WARN("FL: (%ld %ld) '%s'\n", left_reminder_offset, left_reminder_length, raw_text->substr(left_reminder_offset, left_reminder_length).c_str());
|
|
13299
13402
|
#endif
|
|
13300
|
-
it++;
|
|
13301
13403
|
}
|
|
13302
13404
|
|
|
13303
13405
|
// special token
|
|
@@ -13306,16 +13408,25 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<
|
|
|
13306
13408
|
|
|
13307
13409
|
// right
|
|
13308
13410
|
if (match + special_token.length() < raw_text_base_offset + raw_text_base_length) {
|
|
13309
|
-
|
|
13310
|
-
|
|
13311
|
-
|
|
13411
|
+
int64_t right_reminder_offset = match + special_token.length();
|
|
13412
|
+
int64_t right_reminder_length = raw_text_base_length - ((match - raw_text_base_offset) + special_token.length());
|
|
13413
|
+
|
|
13414
|
+
if (data.attr & LLAMA_TOKEN_ATTR_RSTRIP) {
|
|
13415
|
+
while (right_reminder_length > 0 && isspace(raw_text[right_reminder_offset])) {
|
|
13416
|
+
right_reminder_offset++;
|
|
13417
|
+
right_reminder_length--;
|
|
13418
|
+
}
|
|
13419
|
+
}
|
|
13420
|
+
|
|
13421
|
+
if (right_reminder_length > 0) {
|
|
13422
|
+
buffer.emplace_after(it, raw_text, right_reminder_offset, right_reminder_length);
|
|
13423
|
+
it++;
|
|
13424
|
+
}
|
|
13312
13425
|
|
|
13313
13426
|
#ifdef PRETOKENIZERDEBUG
|
|
13314
13427
|
LLAMA_LOG_WARN("FR: (%ld %ld) '%s'\n", right_reminder_offset, right_reminder_length, raw_text->substr(right_reminder_offset, right_reminder_length).c_str());
|
|
13315
13428
|
#endif
|
|
13316
13429
|
|
|
13317
|
-
it++;
|
|
13318
|
-
|
|
13319
13430
|
if (source == 0) {
|
|
13320
13431
|
buffer.erase_after(buffer.before_begin());
|
|
13321
13432
|
} else {
|
|
@@ -13361,9 +13472,7 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
|
|
|
13361
13472
|
// tokenizer.encode('', add_special_tokens=True) returns [1]
|
|
13362
13473
|
// tokenizer.encode('', add_special_tokens=False) returns []
|
|
13363
13474
|
|
|
13364
|
-
static const bool rtrim = true; //TODO: as param
|
|
13365
13475
|
bool is_prev_special = false;
|
|
13366
|
-
bool special_token_rtrim = false;
|
|
13367
13476
|
|
|
13368
13477
|
if (add_special && vocab.special_add_bos != 0) {
|
|
13369
13478
|
GGML_ASSERT(vocab.special_bos_id != -1);
|
|
@@ -13373,25 +13482,8 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
|
|
|
13373
13482
|
|
|
13374
13483
|
for (const auto & fragment : fragment_buffer) {
|
|
13375
13484
|
if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
|
|
13376
|
-
// without adding this leading whitespace, we do not get the same results as the original tokenizer
|
|
13377
|
-
|
|
13378
|
-
// TODO: It's likely possible to get rid of this string copy entirely
|
|
13379
|
-
// by modifying llm_tokenizer_x to operate with string offsets like pre-tokenizer
|
|
13380
|
-
// and passing 'add space prefix' as bool argument
|
|
13381
|
-
//
|
|
13382
13485
|
auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
|
|
13383
13486
|
|
|
13384
|
-
if (special_token_rtrim) {
|
|
13385
|
-
size_t num_whitespaces = 0;
|
|
13386
|
-
while (isspace(raw_text[num_whitespaces])) {
|
|
13387
|
-
num_whitespaces++;
|
|
13388
|
-
}
|
|
13389
|
-
if (num_whitespaces == raw_text.size()) {
|
|
13390
|
-
continue; // skip if all whitespaces
|
|
13391
|
-
}
|
|
13392
|
-
raw_text = raw_text.substr(num_whitespaces);
|
|
13393
|
-
}
|
|
13394
|
-
|
|
13395
13487
|
if (vocab.add_space_prefix) {
|
|
13396
13488
|
if (!output.size() || is_prev_special) { // prefix with space if first token
|
|
13397
13489
|
raw_text = " " + raw_text;
|
|
@@ -13407,11 +13499,6 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
|
|
|
13407
13499
|
} else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
|
|
13408
13500
|
output.push_back(fragment.token);
|
|
13409
13501
|
is_prev_special = true;
|
|
13410
|
-
// phi-3 special tokens without rtrim, works fine for llama-spm too
|
|
13411
|
-
special_token_rtrim = rtrim
|
|
13412
|
-
&& fragment.token != vocab.special_bos_id
|
|
13413
|
-
&& fragment.token != vocab.special_unk_id
|
|
13414
|
-
&& fragment.token != vocab.special_eos_id;
|
|
13415
13502
|
}
|
|
13416
13503
|
}
|
|
13417
13504
|
|
|
@@ -13574,7 +13661,7 @@ static std::pair<bool, const llama_grammar_element *> llama_grammar_match_char(
|
|
|
13574
13661
|
const uint32_t chr) {
|
|
13575
13662
|
|
|
13576
13663
|
bool found = false;
|
|
13577
|
-
bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR;
|
|
13664
|
+
bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR || pos->type == LLAMA_GRETYPE_CHAR_ANY;
|
|
13578
13665
|
|
|
13579
13666
|
GGML_ASSERT(is_positive_char || pos->type == LLAMA_GRETYPE_CHAR_NOT); // NOLINT
|
|
13580
13667
|
|
|
@@ -13583,6 +13670,10 @@ static std::pair<bool, const llama_grammar_element *> llama_grammar_match_char(
|
|
|
13583
13670
|
// inclusive range, e.g. [a-z]
|
|
13584
13671
|
found = found || (pos->value <= chr && chr <= pos[1].value);
|
|
13585
13672
|
pos += 2;
|
|
13673
|
+
} else if (pos->type == LLAMA_GRETYPE_CHAR_ANY) {
|
|
13674
|
+
// Any character matches "."
|
|
13675
|
+
found = true;
|
|
13676
|
+
pos += 1;
|
|
13586
13677
|
} else {
|
|
13587
13678
|
// exact char match, e.g. [a] or "a"
|
|
13588
13679
|
found = found || pos->value == chr;
|
|
@@ -13600,7 +13691,7 @@ static bool llama_grammar_match_partial_char(
|
|
|
13600
13691
|
const llama_grammar_element * pos,
|
|
13601
13692
|
const llama_partial_utf8 partial_utf8) {
|
|
13602
13693
|
|
|
13603
|
-
bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR;
|
|
13694
|
+
bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR || pos->type == LLAMA_GRETYPE_CHAR_ANY;
|
|
13604
13695
|
GGML_ASSERT(is_positive_char || pos->type == LLAMA_GRETYPE_CHAR_NOT);
|
|
13605
13696
|
|
|
13606
13697
|
uint32_t partial_value = partial_utf8.value;
|
|
@@ -13630,6 +13721,9 @@ static bool llama_grammar_match_partial_char(
|
|
|
13630
13721
|
return is_positive_char;
|
|
13631
13722
|
}
|
|
13632
13723
|
pos += 2;
|
|
13724
|
+
} else if (pos->type == LLAMA_GRETYPE_CHAR_ANY) {
|
|
13725
|
+
// Any character matches "."
|
|
13726
|
+
return true;
|
|
13633
13727
|
} else {
|
|
13634
13728
|
// exact char match, e.g. [a] or "a"
|
|
13635
13729
|
if (low <= pos->value && pos->value <= high) {
|
|
@@ -13690,6 +13784,7 @@ static void llama_grammar_advance_stack(
|
|
|
13690
13784
|
}
|
|
13691
13785
|
case LLAMA_GRETYPE_CHAR:
|
|
13692
13786
|
case LLAMA_GRETYPE_CHAR_NOT:
|
|
13787
|
+
case LLAMA_GRETYPE_CHAR_ANY:
|
|
13693
13788
|
if (std::find(new_stacks.begin(), new_stacks.end(), stack) == new_stacks.end()) {
|
|
13694
13789
|
// only add the stack if it's not a duplicate of one we already have
|
|
13695
13790
|
new_stacks.emplace_back(stack);
|
|
@@ -14646,260 +14741,6 @@ void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammar
|
|
|
14646
14741
|
ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
|
|
14647
14742
|
}
|
|
14648
14743
|
|
|
14649
|
-
//
|
|
14650
|
-
// Beam search
|
|
14651
|
-
//
|
|
14652
|
-
|
|
14653
|
-
struct llama_beam {
|
|
14654
|
-
std::vector<llama_token> tokens;
|
|
14655
|
-
float p; // Cumulative beam probability (renormalized relative to all beams)
|
|
14656
|
-
bool eob; // Initialize end-of-beam to false. Callback sets this to true.
|
|
14657
|
-
// Sort beams by probability. In case of ties, prefer beams at eob.
|
|
14658
|
-
bool operator<(const llama_beam & rhs) const {
|
|
14659
|
-
return std::make_pair(p, eob) < std::make_pair(rhs.p, rhs.eob);
|
|
14660
|
-
}
|
|
14661
|
-
// Shift off first n tokens and discard them.
|
|
14662
|
-
void shift_tokens(const size_t n) {
|
|
14663
|
-
if (n) {
|
|
14664
|
-
std::copy(tokens.begin() + n, tokens.end(), tokens.begin());
|
|
14665
|
-
tokens.resize(tokens.size() - n);
|
|
14666
|
-
}
|
|
14667
|
-
}
|
|
14668
|
-
llama_beam_view view() const { return {tokens.data(), tokens.size(), p, eob}; }
|
|
14669
|
-
};
|
|
14670
|
-
|
|
14671
|
-
// A struct for calculating logit-related info.
|
|
14672
|
-
struct llama_logit_info {
|
|
14673
|
-
const float * const logits;
|
|
14674
|
-
const int n_vocab;
|
|
14675
|
-
const float max_l;
|
|
14676
|
-
const float normalizer;
|
|
14677
|
-
struct sum_exp {
|
|
14678
|
-
float max_l;
|
|
14679
|
-
float operator()(float sum, float l) const { return sum + std::exp(l - max_l); }
|
|
14680
|
-
};
|
|
14681
|
-
llama_logit_info(llama_context * ctx)
|
|
14682
|
-
: logits(llama_get_logits(ctx))
|
|
14683
|
-
, n_vocab(llama_n_vocab(llama_get_model(ctx)))
|
|
14684
|
-
, max_l(*std::max_element(logits, logits + n_vocab))
|
|
14685
|
-
, normalizer(1.0f / std::accumulate(logits, logits + n_vocab, 0.0f, sum_exp{max_l}))
|
|
14686
|
-
{ }
|
|
14687
|
-
llama_token_data get_token_data(const llama_token token_id) const {
|
|
14688
|
-
constexpr auto p = std::numeric_limits<float>::quiet_NaN(); // never used
|
|
14689
|
-
return {token_id, logits[token_id], p};
|
|
14690
|
-
}
|
|
14691
|
-
// Return top k token_data by logit.
|
|
14692
|
-
std::vector<llama_token_data> top_k(size_t k) {
|
|
14693
|
-
std::vector<llama_token_data> min_heap; // min-heap by logit
|
|
14694
|
-
const llama_token k_min = std::min(static_cast<llama_token>(k), n_vocab);
|
|
14695
|
-
min_heap.reserve(k_min);
|
|
14696
|
-
for (llama_token token_id = 0 ; token_id < k_min ; ++token_id) {
|
|
14697
|
-
min_heap.push_back(get_token_data(token_id));
|
|
14698
|
-
}
|
|
14699
|
-
auto comp = [](const llama_token_data & a, const llama_token_data & b) { return a.logit > b.logit; };
|
|
14700
|
-
std::make_heap(min_heap.begin(), min_heap.end(), comp);
|
|
14701
|
-
for (llama_token token_id = k_min ; token_id < n_vocab ; ++token_id) {
|
|
14702
|
-
if (min_heap.front().logit < logits[token_id]) {
|
|
14703
|
-
std::pop_heap(min_heap.begin(), min_heap.end(), comp);
|
|
14704
|
-
min_heap.back().id = token_id;
|
|
14705
|
-
min_heap.back().logit = logits[token_id];
|
|
14706
|
-
std::push_heap(min_heap.begin(), min_heap.end(), comp);
|
|
14707
|
-
}
|
|
14708
|
-
}
|
|
14709
|
-
return min_heap;
|
|
14710
|
-
}
|
|
14711
|
-
float probability_from_logit(float logit) const {
|
|
14712
|
-
return normalizer * std::exp(logit - max_l);
|
|
14713
|
-
}
|
|
14714
|
-
};
|
|
14715
|
-
|
|
14716
|
-
struct llama_beam_search_data {
|
|
14717
|
-
llama_context * ctx;
|
|
14718
|
-
size_t n_beams;
|
|
14719
|
-
int n_past;
|
|
14720
|
-
int n_predict;
|
|
14721
|
-
std::vector<llama_beam> beams;
|
|
14722
|
-
std::vector<llama_beam> next_beams;
|
|
14723
|
-
|
|
14724
|
-
// Re-calculated on each loop iteration
|
|
14725
|
-
size_t common_prefix_length;
|
|
14726
|
-
|
|
14727
|
-
// Used to communicate to/from callback on beams state.
|
|
14728
|
-
std::vector<llama_beam_view> beam_views;
|
|
14729
|
-
|
|
14730
|
-
llama_beam_search_data(llama_context * ctx, size_t n_beams, int n_past, int n_predict)
|
|
14731
|
-
: ctx(ctx)
|
|
14732
|
-
, n_beams(n_beams)
|
|
14733
|
-
, n_past(n_past)
|
|
14734
|
-
, n_predict(n_predict)
|
|
14735
|
-
, beam_views(n_beams) {
|
|
14736
|
-
beams.reserve(n_beams);
|
|
14737
|
-
next_beams.reserve(n_beams);
|
|
14738
|
-
}
|
|
14739
|
-
|
|
14740
|
-
// Collapse beams to a single beam given by index.
|
|
14741
|
-
void collapse_beams(const size_t beam_idx) {
|
|
14742
|
-
if (0u < beam_idx) {
|
|
14743
|
-
std::swap(beams[0], beams[beam_idx]);
|
|
14744
|
-
}
|
|
14745
|
-
beams.resize(1);
|
|
14746
|
-
}
|
|
14747
|
-
|
|
14748
|
-
// Min-heaps are used to efficiently collect the top-k elements (k=n_beams).
|
|
14749
|
-
// The repetitive patterns below reflect the 2 stages of heaps:
|
|
14750
|
-
// * Gather elements until the vector is full, then call std::make_heap() on it.
|
|
14751
|
-
// * If the heap is full and a new element is found that should be included, pop the
|
|
14752
|
-
// least element to the back(), replace it with the new, then push it into the heap.
|
|
14753
|
-
void fill_next_beams_by_top_probabilities(llama_beam & beam) {
|
|
14754
|
-
// Min-heaps use a greater-than comparator.
|
|
14755
|
-
const auto comp = [](const llama_beam & a, const llama_beam & b) { return a.p > b.p; };
|
|
14756
|
-
if (beam.eob) {
|
|
14757
|
-
// beam is at end-of-sentence, so just copy it to next_beams if its probability is high enough.
|
|
14758
|
-
if (next_beams.size() < n_beams) {
|
|
14759
|
-
next_beams.push_back(std::move(beam));
|
|
14760
|
-
if (next_beams.size() == n_beams) {
|
|
14761
|
-
std::make_heap(next_beams.begin(), next_beams.end(), comp);
|
|
14762
|
-
}
|
|
14763
|
-
} else if (next_beams.front().p < beam.p) {
|
|
14764
|
-
std::pop_heap(next_beams.begin(), next_beams.end(), comp);
|
|
14765
|
-
next_beams.back() = std::move(beam);
|
|
14766
|
-
std::push_heap(next_beams.begin(), next_beams.end(), comp);
|
|
14767
|
-
}
|
|
14768
|
-
} else {
|
|
14769
|
-
// beam is not at end-of-sentence, so branch with next top_k tokens.
|
|
14770
|
-
if (!beam.tokens.empty()) {
|
|
14771
|
-
llama_decode(ctx, llama_batch_get_one(beam.tokens.data(), beam.tokens.size(), n_past, 0));
|
|
14772
|
-
}
|
|
14773
|
-
llama_logit_info logit_info(ctx);
|
|
14774
|
-
std::vector<llama_token_data> next_tokens = logit_info.top_k(n_beams);
|
|
14775
|
-
|
|
14776
|
-
// Clear the kv slot so that other beams may try different tokens at this position. The llama_decode()
|
|
14777
|
-
// call in loop() will conclusively fill in the kv slot once the beams converge at this position.
|
|
14778
|
-
llama_kv_cache_seq_rm(ctx, 0, n_past, -1);
|
|
14779
|
-
|
|
14780
|
-
size_t i=0;
|
|
14781
|
-
if (next_beams.size() < n_beams) {
|
|
14782
|
-
for (; next_beams.size() < n_beams ; ++i) {
|
|
14783
|
-
llama_beam next_beam = beam;
|
|
14784
|
-
next_beam.tokens.push_back(next_tokens[i].id);
|
|
14785
|
-
next_beam.p *= logit_info.probability_from_logit(next_tokens[i].logit);
|
|
14786
|
-
next_beams.push_back(std::move(next_beam));
|
|
14787
|
-
}
|
|
14788
|
-
std::make_heap(next_beams.begin(), next_beams.end(), comp);
|
|
14789
|
-
} else {
|
|
14790
|
-
for (; next_beams.front().p == 0.0f ; ++i) {
|
|
14791
|
-
std::pop_heap(next_beams.begin(), next_beams.end(), comp);
|
|
14792
|
-
next_beams.back() = beam;
|
|
14793
|
-
next_beams.back().tokens.push_back(next_tokens[i].id);
|
|
14794
|
-
next_beams.back().p *= logit_info.probability_from_logit(next_tokens[i].logit);
|
|
14795
|
-
std::push_heap(next_beams.begin(), next_beams.end(), comp);
|
|
14796
|
-
}
|
|
14797
|
-
}
|
|
14798
|
-
for (; i < n_beams ; ++i) {
|
|
14799
|
-
const float next_p = beam.p * logit_info.probability_from_logit(next_tokens[i].logit);
|
|
14800
|
-
if (next_beams.front().p < next_p) {
|
|
14801
|
-
std::pop_heap(next_beams.begin(), next_beams.end(), comp);
|
|
14802
|
-
next_beams.back() = beam;
|
|
14803
|
-
next_beams.back().tokens.push_back(next_tokens[i].id);
|
|
14804
|
-
next_beams.back().p = next_p;
|
|
14805
|
-
std::push_heap(next_beams.begin(), next_beams.end(), comp);
|
|
14806
|
-
}
|
|
14807
|
-
}
|
|
14808
|
-
}
|
|
14809
|
-
}
|
|
14810
|
-
|
|
14811
|
-
// Find common_prefix_length based on beams.
|
|
14812
|
-
// Requires beams is not empty.
|
|
14813
|
-
size_t find_common_prefix_length() {
|
|
14814
|
-
size_t common_prefix_length = beams[0].tokens.size();
|
|
14815
|
-
for (size_t i = 1 ; i < beams.size() ; ++i) {
|
|
14816
|
-
common_prefix_length = std::min(common_prefix_length, beams[i].tokens.size());
|
|
14817
|
-
for (size_t j = 0 ; j < common_prefix_length ; ++j) {
|
|
14818
|
-
if (beams[0].tokens[j] != beams[i].tokens[j]) {
|
|
14819
|
-
common_prefix_length = j;
|
|
14820
|
-
break;
|
|
14821
|
-
}
|
|
14822
|
-
}
|
|
14823
|
-
}
|
|
14824
|
-
return common_prefix_length;
|
|
14825
|
-
}
|
|
14826
|
-
|
|
14827
|
-
// Construct beams_state to send back to caller via the callback function.
|
|
14828
|
-
// Side effect: set common_prefix_length = find_common_prefix_length();
|
|
14829
|
-
llama_beams_state get_beams_state(const bool last_call) {
|
|
14830
|
-
for (size_t i = 0 ; i < beams.size() ; ++i) {
|
|
14831
|
-
beam_views[i] = beams[i].view();
|
|
14832
|
-
}
|
|
14833
|
-
common_prefix_length = find_common_prefix_length();
|
|
14834
|
-
return {beam_views.data(), beams.size(), common_prefix_length, last_call};
|
|
14835
|
-
}
|
|
14836
|
-
|
|
14837
|
-
// Loop:
|
|
14838
|
-
// * while i < n_predict, AND
|
|
14839
|
-
// * any of the beams have not yet reached end-of-beam (eob), AND
|
|
14840
|
-
// * the highest probability beam(s) (plural in case of ties) are not at end-of-sentence
|
|
14841
|
-
// (since all other beam probabilities can only decrease)
|
|
14842
|
-
void loop(const llama_beam_search_callback_fn_t callback, void * const callback_data) {
|
|
14843
|
-
beams.push_back({{}, 1.0f, false}); // Start with one empty beam w/ probability = 1.0 and !eob.
|
|
14844
|
-
const auto not_eob = [](const llama_beam & beam) { return !beam.eob; };
|
|
14845
|
-
for (int i = 0 ; i < n_predict && std::any_of(beams.begin(),beams.end(),not_eob) &&
|
|
14846
|
-
!beams[top_beam_index()].eob ; ++i) {
|
|
14847
|
-
callback(callback_data, get_beams_state(false)); // Sets common_prefix_length
|
|
14848
|
-
update_beams_from_beam_views(); // Update values (p,eob) that callback may have changed.
|
|
14849
|
-
if (common_prefix_length) {
|
|
14850
|
-
llama_decode(ctx, llama_batch_get_one(beams[0].tokens.data(), common_prefix_length, n_past, 0));
|
|
14851
|
-
n_past += common_prefix_length;
|
|
14852
|
-
}
|
|
14853
|
-
// Zero-out next_beam probabilities to place them last in following min-heap.
|
|
14854
|
-
std::for_each(next_beams.begin(), next_beams.end(), [](llama_beam & beam) { beam.p = 0.0f; });
|
|
14855
|
-
for (llama_beam & beam : beams) {
|
|
14856
|
-
beam.shift_tokens(common_prefix_length);
|
|
14857
|
-
fill_next_beams_by_top_probabilities(beam);
|
|
14858
|
-
}
|
|
14859
|
-
// next_beams become the beams of next/final iteration. Swap them to re-use memory.
|
|
14860
|
-
beams.swap(next_beams);
|
|
14861
|
-
renormalize_beam_probabilities(beams);
|
|
14862
|
-
}
|
|
14863
|
-
collapse_beams(top_beam_index());
|
|
14864
|
-
callback(callback_data, get_beams_state(true));
|
|
14865
|
-
}
|
|
14866
|
-
|
|
14867
|
-
// As beams grow, the cumulative probabilities decrease.
|
|
14868
|
-
// Renormalize them to avoid floating point underflow.
|
|
14869
|
-
static void renormalize_beam_probabilities(std::vector<llama_beam> & beams) {
|
|
14870
|
-
const auto sum_p = [](float sum, llama_beam & beam) { return sum + beam.p; };
|
|
14871
|
-
const float inv_sum = 1.0f / std::accumulate(beams.begin(), beams.end(), 0.0f, sum_p);
|
|
14872
|
-
std::for_each(beams.begin(), beams.end(), [=](llama_beam & beam) { beam.p *= inv_sum; });
|
|
14873
|
-
}
|
|
14874
|
-
|
|
14875
|
-
// Assumes beams is non-empty. Uses llama_beam::operator<() for ordering.
|
|
14876
|
-
size_t top_beam_index() {
|
|
14877
|
-
return std::max_element(beams.begin(), beams.end()) - beams.begin();
|
|
14878
|
-
}
|
|
14879
|
-
|
|
14880
|
-
// Copy (p,eob) for each beam which may have been changed by the callback.
|
|
14881
|
-
void update_beams_from_beam_views() {
|
|
14882
|
-
for (size_t i = 0 ; i < beams.size() ; ++i) {
|
|
14883
|
-
beams[i].p = beam_views[i].p;
|
|
14884
|
-
beams[i].eob = beam_views[i].eob;
|
|
14885
|
-
}
|
|
14886
|
-
}
|
|
14887
|
-
};
|
|
14888
|
-
|
|
14889
|
-
void llama_beam_search(llama_context * ctx,
|
|
14890
|
-
llama_beam_search_callback_fn_t callback, void * callback_data,
|
|
14891
|
-
size_t n_beams, int n_past, int n_predict) {
|
|
14892
|
-
assert(ctx);
|
|
14893
|
-
const int64_t t_start_sample_us = ggml_time_us();
|
|
14894
|
-
|
|
14895
|
-
llama_beam_search_data beam_search_data(ctx, n_beams, n_past, n_predict);
|
|
14896
|
-
|
|
14897
|
-
beam_search_data.loop(callback, callback_data);
|
|
14898
|
-
|
|
14899
|
-
ctx->t_sample_us += ggml_time_us() - t_start_sample_us;
|
|
14900
|
-
ctx->n_sample++;
|
|
14901
|
-
}
|
|
14902
|
-
|
|
14903
14744
|
//
|
|
14904
14745
|
// quantization
|
|
14905
14746
|
//
|
|
@@ -15417,6 +15258,14 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
|
|
|
15417
15258
|
if (imatrix_data) {
|
|
15418
15259
|
LLAMA_LOG_INFO("================================ Have weights data with %d entries\n",int(imatrix_data->size()));
|
|
15419
15260
|
qs.has_imatrix = true;
|
|
15261
|
+
// check imatrix for nans or infs
|
|
15262
|
+
for (const auto & kv : *imatrix_data) {
|
|
15263
|
+
for (float f : kv.second) {
|
|
15264
|
+
if (!std::isfinite(f)) {
|
|
15265
|
+
throw std::runtime_error(format("imatrix contains non-finite value %f\n", f));
|
|
15266
|
+
}
|
|
15267
|
+
}
|
|
15268
|
+
}
|
|
15420
15269
|
}
|
|
15421
15270
|
}
|
|
15422
15271
|
|
|
@@ -16110,7 +15959,7 @@ bool llama_supports_mlock(void) {
|
|
|
16110
15959
|
}
|
|
16111
15960
|
|
|
16112
15961
|
bool llama_supports_gpu_offload(void) {
|
|
16113
|
-
#if defined(GGML_USE_CUDA) || defined(
|
|
15962
|
+
#if defined(GGML_USE_CUDA) || defined(GGML_USE_METAL) || defined(GGML_USE_VULKAN) || \
|
|
16114
15963
|
defined(GGML_USE_SYCL) || defined(GGML_USE_KOMPUTE) || defined(GGML_USE_RPC)
|
|
16115
15964
|
// Defined when llama.cpp is compiled with support for offloading model layers to GPU.
|
|
16116
15965
|
return true;
|
|
@@ -16167,7 +16016,7 @@ struct llama_model * llama_load_model_from_file(
|
|
|
16167
16016
|
return true;
|
|
16168
16017
|
};
|
|
16169
16018
|
}
|
|
16170
|
-
if (params.rpc_servers != nullptr) {
|
|
16019
|
+
if (params.rpc_servers != nullptr && params.rpc_servers[0] != '\0') {
|
|
16171
16020
|
// split the servers set them into model->rpc_servers
|
|
16172
16021
|
std::string servers(params.rpc_servers);
|
|
16173
16022
|
size_t pos = 0;
|
|
@@ -16221,6 +16070,11 @@ struct llama_context * llama_new_context_with_model(
|
|
|
16221
16070
|
params.flash_attn = false;
|
|
16222
16071
|
}
|
|
16223
16072
|
|
|
16073
|
+
if (params.type_v != GGML_TYPE_F16 && !params.flash_attn) {
|
|
16074
|
+
LLAMA_LOG_ERROR("%s: V cache quantization requires flash_attn\n", __func__);
|
|
16075
|
+
return nullptr;
|
|
16076
|
+
}
|
|
16077
|
+
|
|
16224
16078
|
llama_context * ctx = new llama_context(*model);
|
|
16225
16079
|
|
|
16226
16080
|
const auto & hparams = model->hparams;
|
|
@@ -16259,8 +16113,8 @@ struct llama_context * llama_new_context_with_model(
|
|
|
16259
16113
|
|
|
16260
16114
|
cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch);
|
|
16261
16115
|
|
|
16262
|
-
cparams.
|
|
16263
|
-
hparams.
|
|
16116
|
+
cparams.n_ctx_orig_yarn = params.yarn_orig_ctx != 0 ? params.yarn_orig_ctx :
|
|
16117
|
+
hparams.n_ctx_orig_yarn != 0 ? hparams.n_ctx_orig_yarn :
|
|
16264
16118
|
hparams.n_ctx_train;
|
|
16265
16119
|
|
|
16266
16120
|
cparams.cb_eval = params.cb_eval;
|
|
@@ -16325,17 +16179,7 @@ struct llama_context * llama_new_context_with_model(
|
|
|
16325
16179
|
|
|
16326
16180
|
if (!hparams.vocab_only) {
|
|
16327
16181
|
// initialize backends
|
|
16328
|
-
#if defined(
|
|
16329
|
-
for (auto & server : model->rpc_servers) {
|
|
16330
|
-
ggml_backend_t backend = ggml_backend_rpc_init(server.c_str());
|
|
16331
|
-
if (backend == nullptr) {
|
|
16332
|
-
LLAMA_LOG_ERROR("%s: failed to connect RPC backend to %s\n", __func__, server.c_str());
|
|
16333
|
-
llama_free(ctx);
|
|
16334
|
-
return nullptr;
|
|
16335
|
-
}
|
|
16336
|
-
ctx->backends.push_back(backend);
|
|
16337
|
-
}
|
|
16338
|
-
#elif defined(GGML_USE_METAL)
|
|
16182
|
+
#if defined(GGML_USE_METAL)
|
|
16339
16183
|
if (model->n_gpu_layers > 0) {
|
|
16340
16184
|
ctx->backend_metal = ggml_backend_metal_init();
|
|
16341
16185
|
if (ctx->backend_metal == nullptr) {
|
|
@@ -16374,7 +16218,7 @@ struct llama_context * llama_new_context_with_model(
|
|
|
16374
16218
|
return nullptr;
|
|
16375
16219
|
}
|
|
16376
16220
|
if (model->split_mode == LLAMA_SPLIT_MODE_NONE) {
|
|
16377
|
-
ggml_backend_t backend = ggml_backend_vk_init(
|
|
16221
|
+
ggml_backend_t backend = ggml_backend_vk_init(model->main_gpu);
|
|
16378
16222
|
if (backend == nullptr) {
|
|
16379
16223
|
LLAMA_LOG_ERROR("%s: failed to initialize Vulkan backend\n", __func__);
|
|
16380
16224
|
llama_free(ctx);
|
|
@@ -16428,6 +16272,29 @@ struct llama_context * llama_new_context_with_model(
|
|
|
16428
16272
|
ctx->backends.push_back(backend);
|
|
16429
16273
|
}
|
|
16430
16274
|
#endif
|
|
16275
|
+
|
|
16276
|
+
#ifdef GGML_USE_BLAS
|
|
16277
|
+
ctx->backend_blas = ggml_backend_blas_init();
|
|
16278
|
+
if (ctx->backend_blas == nullptr) {
|
|
16279
|
+
LLAMA_LOG_WARN("%s: failed to initialize BLAS backend\n", __func__);
|
|
16280
|
+
} else {
|
|
16281
|
+
ctx->backends.push_back(ctx->backend_blas);
|
|
16282
|
+
}
|
|
16283
|
+
#endif
|
|
16284
|
+
|
|
16285
|
+
#if defined(GGML_USE_RPC)
|
|
16286
|
+
if (model->n_gpu_layers > 0) {
|
|
16287
|
+
for (const auto & endpoint : model->rpc_servers) {
|
|
16288
|
+
ggml_backend_t backend = ggml_backend_rpc_init(endpoint.c_str());
|
|
16289
|
+
if (backend == nullptr) {
|
|
16290
|
+
LLAMA_LOG_ERROR("%s: failed to initialize RPC to '%s'\n", __func__, endpoint.c_str());
|
|
16291
|
+
llama_free(ctx);
|
|
16292
|
+
return nullptr;
|
|
16293
|
+
}
|
|
16294
|
+
ctx->backends.push_back(backend);
|
|
16295
|
+
}
|
|
16296
|
+
}
|
|
16297
|
+
#endif
|
|
16431
16298
|
ctx->backend_cpu = ggml_backend_cpu_init();
|
|
16432
16299
|
if (ctx->backend_cpu == nullptr) {
|
|
16433
16300
|
LLAMA_LOG_ERROR("%s: failed to initialize CPU backend\n", __func__);
|
|
@@ -18209,9 +18076,9 @@ float llama_token_get_score(const struct llama_model * model, llama_token token)
|
|
|
18209
18076
|
return model->vocab.id_to_token[token].score;
|
|
18210
18077
|
}
|
|
18211
18078
|
|
|
18212
|
-
|
|
18079
|
+
llama_token_attr llama_token_get_attr(const struct llama_model * model, llama_token token) {
|
|
18213
18080
|
GGML_ASSERT(model->vocab.type != LLAMA_VOCAB_TYPE_NONE);
|
|
18214
|
-
return model->vocab.id_to_token[token].
|
|
18081
|
+
return model->vocab.id_to_token[token].attr;
|
|
18215
18082
|
}
|
|
18216
18083
|
|
|
18217
18084
|
bool llama_token_is_eog(const struct llama_model * model, llama_token token) {
|
|
@@ -18313,9 +18180,14 @@ static std::string llama_decode_text(const std::string & text) {
|
|
|
18313
18180
|
|
|
18314
18181
|
// does not write null-terminator to buf
|
|
18315
18182
|
int32_t llama_token_to_piece(const struct llama_model * model, llama_token token, char * buf, int32_t length, bool special) {
|
|
18183
|
+
// ref: https://github.com/ggerganov/llama.cpp/pull/7587#discussion_r1620983843
|
|
18184
|
+
if (!special && llama_is_control_token(model->vocab, token)) {
|
|
18185
|
+
return 0;
|
|
18186
|
+
}
|
|
18187
|
+
|
|
18316
18188
|
// if we have a cache - use it
|
|
18317
18189
|
{
|
|
18318
|
-
const auto & cache =
|
|
18190
|
+
const auto & cache = model->vocab.cache_token_to_piece;
|
|
18319
18191
|
|
|
18320
18192
|
if (!cache.empty()) {
|
|
18321
18193
|
const auto & res = cache.at(token);
|