llama_cpp 0.16.0 → 0.16.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/ext/llama_cpp/extconf.rb +3 -0
- data/ext/llama_cpp/llama_cpp.cpp +14 -0
- data/lib/llama_cpp/version.rb +2 -2
- data/sig/llama_cpp.rbs +4 -0
- data/vendor/tmp/llama.cpp/Makefile +119 -54
- data/vendor/tmp/llama.cpp/ggml-alloc.c +78 -22
- data/vendor/tmp/llama.cpp/ggml-backend-impl.h +20 -8
- data/vendor/tmp/llama.cpp/ggml-backend.c +190 -65
- data/vendor/tmp/llama.cpp/ggml-backend.h +6 -3
- data/vendor/tmp/llama.cpp/ggml-blas.cpp +363 -0
- data/vendor/tmp/llama.cpp/ggml-blas.h +23 -0
- data/vendor/tmp/llama.cpp/ggml-common.h +6 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/argsort.cu +1 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/dmmv.cu +21 -9
- data/vendor/tmp/llama.cpp/ggml-cuda/fattn-tile-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/mmq.cu +15 -1491
- data/vendor/tmp/llama.cpp/ggml-cuda/mmvq.cu +77 -62
- data/vendor/tmp/llama.cpp/ggml-cuda/quantize.cu +77 -10
- data/vendor/tmp/llama.cpp/ggml-cuda/softmax.cu +1 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb32.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb16.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb32.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb8.cu +1 -1
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/vendor/tmp/llama.cpp/ggml-cuda/unary.cu +48 -0
- data/vendor/tmp/llama.cpp/ggml-cuda.cu +95 -129
- data/vendor/tmp/llama.cpp/ggml-impl.h +1 -1
- data/vendor/tmp/llama.cpp/ggml-kompute.cpp +8 -7
- data/vendor/tmp/llama.cpp/ggml-metal.m +17 -9
- data/vendor/tmp/llama.cpp/ggml-quants.c +982 -368
- data/vendor/tmp/llama.cpp/ggml-rpc.cpp +21 -15
- data/vendor/tmp/llama.cpp/ggml-sycl.cpp +2133 -13215
- data/vendor/tmp/llama.cpp/ggml-sycl.h +1 -10
- data/vendor/tmp/llama.cpp/ggml-vulkan-shaders.hpp +28826 -25037
- data/vendor/tmp/llama.cpp/ggml-vulkan.cpp +438 -493
- data/vendor/tmp/llama.cpp/ggml.c +158 -414
- data/vendor/tmp/llama.cpp/ggml.h +6 -0
- data/vendor/tmp/llama.cpp/llama.cpp +628 -279
- data/vendor/tmp/llama.cpp/llama.h +9 -1
- data/vendor/tmp/llama.cpp/sgemm.cpp +2 -0
- data/vendor/tmp/llama.cpp/unicode-data.cpp +851 -801
- data/vendor/tmp/llama.cpp/unicode.cpp +33 -19
- data/vendor/tmp/llama.cpp/unicode.h +1 -1
- metadata +15 -3
|
@@ -21,6 +21,10 @@
|
|
|
21
21
|
# include "ggml-kompute.h"
|
|
22
22
|
#endif
|
|
23
23
|
|
|
24
|
+
#ifdef GGML_USE_BLAS
|
|
25
|
+
# include "ggml-blas.h"
|
|
26
|
+
#endif
|
|
27
|
+
|
|
24
28
|
#ifdef GGML_USE_METAL
|
|
25
29
|
# include "ggml-metal.h"
|
|
26
30
|
#endif
|
|
@@ -282,6 +286,7 @@ enum llm_kv {
|
|
|
282
286
|
LLM_KV_LEADING_DENSE_BLOCK_COUNT,
|
|
283
287
|
LLM_KV_FEED_FORWARD_LENGTH,
|
|
284
288
|
LLM_KV_EXPERT_FEED_FORWARD_LENGTH,
|
|
289
|
+
LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH,
|
|
285
290
|
LLM_KV_USE_PARALLEL_RESIDUAL,
|
|
286
291
|
LLM_KV_TENSOR_DATA_LAYOUT,
|
|
287
292
|
LLM_KV_EXPERT_COUNT,
|
|
@@ -360,21 +365,22 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
|
|
|
360
365
|
{ LLM_KV_GENERAL_SOURCE_URL, "general.source.url" },
|
|
361
366
|
{ LLM_KV_GENERAL_SOURCE_HF_REPO, "general.source.huggingface.repository" },
|
|
362
367
|
|
|
363
|
-
{ LLM_KV_VOCAB_SIZE,
|
|
364
|
-
{ LLM_KV_CONTEXT_LENGTH,
|
|
365
|
-
{ LLM_KV_EMBEDDING_LENGTH,
|
|
366
|
-
{ LLM_KV_BLOCK_COUNT,
|
|
367
|
-
{ LLM_KV_LEADING_DENSE_BLOCK_COUNT,
|
|
368
|
-
{ LLM_KV_FEED_FORWARD_LENGTH,
|
|
369
|
-
{ LLM_KV_EXPERT_FEED_FORWARD_LENGTH,
|
|
370
|
-
{
|
|
371
|
-
{
|
|
372
|
-
{
|
|
373
|
-
{
|
|
374
|
-
{
|
|
375
|
-
{
|
|
376
|
-
{
|
|
377
|
-
{
|
|
368
|
+
{ LLM_KV_VOCAB_SIZE, "%s.vocab_size" },
|
|
369
|
+
{ LLM_KV_CONTEXT_LENGTH, "%s.context_length" },
|
|
370
|
+
{ LLM_KV_EMBEDDING_LENGTH, "%s.embedding_length" },
|
|
371
|
+
{ LLM_KV_BLOCK_COUNT, "%s.block_count" },
|
|
372
|
+
{ LLM_KV_LEADING_DENSE_BLOCK_COUNT, "%s.leading_dense_block_count" },
|
|
373
|
+
{ LLM_KV_FEED_FORWARD_LENGTH, "%s.feed_forward_length" },
|
|
374
|
+
{ LLM_KV_EXPERT_FEED_FORWARD_LENGTH, "%s.expert_feed_forward_length" },
|
|
375
|
+
{ LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, "%s.expert_shared_feed_forward_length" },
|
|
376
|
+
{ LLM_KV_USE_PARALLEL_RESIDUAL, "%s.use_parallel_residual" },
|
|
377
|
+
{ LLM_KV_TENSOR_DATA_LAYOUT, "%s.tensor_data_layout" },
|
|
378
|
+
{ LLM_KV_EXPERT_COUNT, "%s.expert_count" },
|
|
379
|
+
{ LLM_KV_EXPERT_USED_COUNT, "%s.expert_used_count" },
|
|
380
|
+
{ LLM_KV_EXPERT_SHARED_COUNT, "%s.expert_shared_count" },
|
|
381
|
+
{ LLM_KV_EXPERT_WEIGHTS_SCALE, "%s.expert_weights_scale" },
|
|
382
|
+
{ LLM_KV_POOLING_TYPE , "%s.pooling_type" },
|
|
383
|
+
{ LLM_KV_LOGIT_SCALE, "%s.logit_scale" },
|
|
378
384
|
|
|
379
385
|
{ LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" },
|
|
380
386
|
{ LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" },
|
|
@@ -704,6 +710,7 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA
|
|
|
704
710
|
{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },
|
|
705
711
|
{ LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" },
|
|
706
712
|
{ LLM_TENSOR_TOKEN_TYPES, "token_types" },
|
|
713
|
+
{ LLM_TENSOR_ATTN_NORM_2, "blk.%d.attn_norm_2" },
|
|
707
714
|
{ LLM_TENSOR_ATTN_OUT_NORM, "blk.%d.attn_output_norm" },
|
|
708
715
|
{ LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
|
|
709
716
|
{ LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm" },
|
|
@@ -1273,6 +1280,126 @@ struct no_init {
|
|
|
1273
1280
|
};
|
|
1274
1281
|
|
|
1275
1282
|
struct llama_file {
|
|
1283
|
+
|
|
1284
|
+
#if defined(_WIN32)
|
|
1285
|
+
// use FILE * so we don't have to re-open the file to mmap
|
|
1286
|
+
FILE * fp;
|
|
1287
|
+
HANDLE fp_win32;
|
|
1288
|
+
size_t size;
|
|
1289
|
+
|
|
1290
|
+
private:
|
|
1291
|
+
std::string GetErrorMessageWin32(DWORD error_code) const {
|
|
1292
|
+
std::string ret;
|
|
1293
|
+
LPSTR lpMsgBuf = NULL;
|
|
1294
|
+
DWORD bufLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
|
|
1295
|
+
NULL, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&lpMsgBuf, 0, NULL);
|
|
1296
|
+
if (!bufLen) {
|
|
1297
|
+
ret = format("Win32 error code: %s", error_code);
|
|
1298
|
+
} else {
|
|
1299
|
+
ret = lpMsgBuf;
|
|
1300
|
+
LocalFree(lpMsgBuf);
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
return ret;
|
|
1304
|
+
}
|
|
1305
|
+
|
|
1306
|
+
public:
|
|
1307
|
+
|
|
1308
|
+
llama_file(const char * fname, const char * mode) {
|
|
1309
|
+
fp = ggml_fopen(fname, mode);
|
|
1310
|
+
if (fp == NULL) {
|
|
1311
|
+
throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
|
|
1312
|
+
}
|
|
1313
|
+
fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));
|
|
1314
|
+
seek(0, SEEK_END);
|
|
1315
|
+
size = tell();
|
|
1316
|
+
seek(0, SEEK_SET);
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
size_t tell() const {
|
|
1320
|
+
// SetFilePointerEx returns the current position when seeking relative 0 bytes
|
|
1321
|
+
LARGE_INTEGER li;
|
|
1322
|
+
li.QuadPart = 0;
|
|
1323
|
+
BOOL ret = SetFilePointerEx(fp_win32, li, &li, FILE_CURRENT);
|
|
1324
|
+
if (!ret) {
|
|
1325
|
+
throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
return li.QuadPart;
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
void seek(size_t offset, int whence) const {
|
|
1332
|
+
// no need to convert SEEK_* to FILE_*. The enums are the same.
|
|
1333
|
+
// Still, keep static asserts to avoid failures in the future.
|
|
1334
|
+
static_assert(SEEK_SET == FILE_BEGIN, "SEEK_SET != FILE_BEGIN");
|
|
1335
|
+
static_assert(SEEK_CUR == FILE_CURRENT, "SEEK_CUR != FILE_CURRENT");
|
|
1336
|
+
static_assert(SEEK_END == FILE_END, "SEEK_END != FILE_END");
|
|
1337
|
+
|
|
1338
|
+
LARGE_INTEGER li;
|
|
1339
|
+
li.QuadPart = offset;
|
|
1340
|
+
BOOL ret = SetFilePointerEx(fp_win32, li, NULL, whence);
|
|
1341
|
+
if (!ret) {
|
|
1342
|
+
throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
|
|
1346
|
+
void read_raw(void * ptr, size_t len) const {
|
|
1347
|
+
// On Win32 ReadFile is significant faster than fread which is again significant faster than std::fstream. Thus
|
|
1348
|
+
// use the Win32 API to do file io instead of the C/C++ library functions.
|
|
1349
|
+
|
|
1350
|
+
// There are conditions under which ReadFile cannot read chunks >64MB.
|
|
1351
|
+
// Thus split the operation into smaller chunks if len exceeds this limit.
|
|
1352
|
+
size_t bytes_read = 0;
|
|
1353
|
+
while (bytes_read < len) {
|
|
1354
|
+
size_t chunk_size = std::min<size_t>(len - bytes_read, 64*1024*1024);
|
|
1355
|
+
DWORD chunk_read = 0;
|
|
1356
|
+
BOOL result = ReadFile(fp_win32, reinterpret_cast<char*>(ptr) + bytes_read, chunk_size, &chunk_read, NULL);
|
|
1357
|
+
if (!result) {
|
|
1358
|
+
throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
|
|
1359
|
+
}
|
|
1360
|
+
if (chunk_read < chunk_size || chunk_read == 0) {
|
|
1361
|
+
throw std::runtime_error("unexpectedly reached end of file");
|
|
1362
|
+
}
|
|
1363
|
+
|
|
1364
|
+
bytes_read += chunk_read;
|
|
1365
|
+
} ;
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
uint32_t read_u32() const {
|
|
1369
|
+
uint32_t val;
|
|
1370
|
+
read_raw(&val, sizeof(val));
|
|
1371
|
+
return val;
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
void write_raw(const void * ptr, size_t len) const {
|
|
1375
|
+
// There are conditions under which WriteFile cannot write chunks >64MB.
|
|
1376
|
+
// Thus split the operation into smaller chunks if len exceeds this limit.
|
|
1377
|
+
size_t bytes_written = 0;
|
|
1378
|
+
while (bytes_written < len) {
|
|
1379
|
+
size_t chunk_size = std::min<size_t>(len - bytes_written, 64*1024*1024);
|
|
1380
|
+
DWORD chunk_written = 0;
|
|
1381
|
+
BOOL result = WriteFile(fp_win32, reinterpret_cast<char const*>(ptr) + bytes_written, chunk_size, &chunk_written, NULL);
|
|
1382
|
+
if (!result) {
|
|
1383
|
+
throw std::runtime_error(format("write error: %s", GetErrorMessageWin32(GetLastError()).c_str()));
|
|
1384
|
+
}
|
|
1385
|
+
if (chunk_written < chunk_size || chunk_written == 0) {
|
|
1386
|
+
throw std::runtime_error("unexpectedly failed to write bytes");
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
bytes_written += chunk_written;
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
void write_u32(std::uint32_t val) const {
|
|
1394
|
+
write_raw(&val, sizeof(val));
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1397
|
+
~llama_file() {
|
|
1398
|
+
if (fp) {
|
|
1399
|
+
std::fclose(fp);
|
|
1400
|
+
}
|
|
1401
|
+
}
|
|
1402
|
+
#else
|
|
1276
1403
|
// use FILE * so we don't have to re-open the file to mmap
|
|
1277
1404
|
FILE * fp;
|
|
1278
1405
|
size_t size;
|
|
@@ -1293,7 +1420,10 @@ struct llama_file {
|
|
|
1293
1420
|
#else
|
|
1294
1421
|
long ret = std::ftell(fp);
|
|
1295
1422
|
#endif
|
|
1296
|
-
|
|
1423
|
+
if (ret == -1) {
|
|
1424
|
+
throw std::runtime_error(format("ftell error: %s", strerror(errno)));
|
|
1425
|
+
}
|
|
1426
|
+
|
|
1297
1427
|
return (size_t) ret;
|
|
1298
1428
|
}
|
|
1299
1429
|
|
|
@@ -1303,7 +1433,9 @@ struct llama_file {
|
|
|
1303
1433
|
#else
|
|
1304
1434
|
int ret = std::fseek(fp, (long) offset, whence);
|
|
1305
1435
|
#endif
|
|
1306
|
-
|
|
1436
|
+
if (ret != 0) {
|
|
1437
|
+
throw std::runtime_error(format("seek error: %s", strerror(errno)));
|
|
1438
|
+
}
|
|
1307
1439
|
}
|
|
1308
1440
|
|
|
1309
1441
|
void read_raw(void * ptr, size_t len) const {
|
|
@@ -1346,6 +1478,7 @@ struct llama_file {
|
|
|
1346
1478
|
std::fclose(fp);
|
|
1347
1479
|
}
|
|
1348
1480
|
}
|
|
1481
|
+
#endif
|
|
1349
1482
|
};
|
|
1350
1483
|
using llama_files = std::vector<std::unique_ptr<llama_file>>;
|
|
1351
1484
|
|
|
@@ -1839,6 +1972,7 @@ struct llama_hparams {
|
|
|
1839
1972
|
uint32_t n_lora_q = 0;
|
|
1840
1973
|
uint32_t n_lora_kv = 0;
|
|
1841
1974
|
uint32_t n_ff_exp = 0;
|
|
1975
|
+
uint32_t n_ff_shexp = 0;
|
|
1842
1976
|
uint32_t n_expert_shared = 0;
|
|
1843
1977
|
float expert_weights_scale = 0.0;
|
|
1844
1978
|
|
|
@@ -1887,6 +2021,7 @@ struct llama_hparams {
|
|
|
1887
2021
|
if (this->n_lora_q != other.n_lora_q) return true;
|
|
1888
2022
|
if (this->n_lora_kv != other.n_lora_kv) return true;
|
|
1889
2023
|
if (this->n_ff_exp != other.n_ff_exp) return true;
|
|
2024
|
+
if (this->n_ff_shexp != other.n_ff_shexp) return true;
|
|
1890
2025
|
if (this->n_expert_shared != other.n_expert_shared) return true;
|
|
1891
2026
|
|
|
1892
2027
|
if (this->rope_finetuned != other.rope_finetuned) return true;
|
|
@@ -2158,6 +2293,8 @@ struct llama_vocab {
|
|
|
2158
2293
|
enum llama_vocab_type type = LLAMA_VOCAB_TYPE_SPM;
|
|
2159
2294
|
enum llama_vocab_pre_type type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
|
|
2160
2295
|
|
|
2296
|
+
int max_token_len = 0; // used for optimizing longest token search
|
|
2297
|
+
|
|
2161
2298
|
std::unordered_map<token, id> token_to_id;
|
|
2162
2299
|
std::vector<token_data> id_to_token;
|
|
2163
2300
|
|
|
@@ -2175,16 +2312,17 @@ struct llama_vocab {
|
|
|
2175
2312
|
id special_cls_id = -1;
|
|
2176
2313
|
id special_mask_id = -1;
|
|
2177
2314
|
|
|
2178
|
-
int special_add_bos = -1; // -1 unknown, 1 add, 0 don't add.
|
|
2179
|
-
int special_add_eos = -1; // -1 unknown, 1 add, 0 don't add.
|
|
2180
|
-
|
|
2181
2315
|
id linefeed_id = 13;
|
|
2182
2316
|
id special_prefix_id = -1;
|
|
2183
2317
|
id special_suffix_id = -1;
|
|
2184
2318
|
id special_middle_id = -1;
|
|
2185
2319
|
id special_eot_id = -1; // TODO: move above after "eos_id", and here add "file separator" token
|
|
2186
2320
|
|
|
2187
|
-
|
|
2321
|
+
// tokenizer flags
|
|
2322
|
+
bool tokenizer_add_space_prefix = true;
|
|
2323
|
+
bool tokenizer_add_bos = false;
|
|
2324
|
+
bool tokenizer_add_eos = false;
|
|
2325
|
+
bool tokenizer_ignore_merges = false;
|
|
2188
2326
|
|
|
2189
2327
|
int find_bpe_rank(const std::string & token_left, const std::string & token_right) const {
|
|
2190
2328
|
GGML_ASSERT(token_left.find(' ') == std::string::npos);
|
|
@@ -2298,9 +2436,13 @@ struct llama_context {
|
|
|
2298
2436
|
std::vector<ggml_backend_t> backends;
|
|
2299
2437
|
#ifdef GGML_USE_METAL
|
|
2300
2438
|
ggml_backend_t backend_metal = nullptr;
|
|
2439
|
+
#endif
|
|
2440
|
+
#ifdef GGML_USE_BLAS
|
|
2441
|
+
ggml_backend_t backend_blas = nullptr;
|
|
2301
2442
|
#endif
|
|
2302
2443
|
ggml_backend_t backend_cpu = nullptr;
|
|
2303
2444
|
|
|
2445
|
+
|
|
2304
2446
|
const llama_model & model;
|
|
2305
2447
|
|
|
2306
2448
|
// key + value cache for the self attention
|
|
@@ -3712,6 +3854,44 @@ struct llama_model_loader {
|
|
|
3712
3854
|
std::vector<no_init<uint8_t>> read_buf;
|
|
3713
3855
|
std::vector<std::future<std::pair<ggml_tensor *, bool>>> validation_result;
|
|
3714
3856
|
|
|
3857
|
+
#if defined(GGML_USE_CUDA)
|
|
3858
|
+
// 4 staging buffers for async uploads, each sized 1MB seems to be a good default for single NVMe drives.
|
|
3859
|
+
// NVMe raid configurations might require more / larger buffers.
|
|
3860
|
+
constexpr size_t num_buffers = 4;
|
|
3861
|
+
constexpr size_t buffer_size = 1 * 1024 * 1024; // 1MB
|
|
3862
|
+
|
|
3863
|
+
std::vector<ggml_backend_buffer_t> host_buffers;
|
|
3864
|
+
std::vector<void*> host_ptrs;
|
|
3865
|
+
std::vector<ggml_backend_event_t> events;
|
|
3866
|
+
size_t buffer_idx = 0; // buffer to use for async loads
|
|
3867
|
+
|
|
3868
|
+
ggml_backend_t cuda_backend = nullptr;
|
|
3869
|
+
if (!use_mmap && !check_tensors) {
|
|
3870
|
+
// When not using mmaped io use async uploads from pinned memory to GPU memory.
|
|
3871
|
+
// First determine if the CUDA backend is active, and if so, determine the device ID.
|
|
3872
|
+
ggml_backend_buffer_t buf = bufs_mmap.count(0) ? bufs_mmap.at(0) : nullptr;
|
|
3873
|
+
if (buf) {
|
|
3874
|
+
ggml_backend_buffer_type_t buffer_type = ggml_backend_buffer_get_type(buf);
|
|
3875
|
+
for (int i = 0; i < ggml_backend_cuda_get_device_count(); ++i) {
|
|
3876
|
+
auto * cuda_buffer_type = ggml_backend_cuda_buffer_type(i);
|
|
3877
|
+
if (buffer_type == cuda_buffer_type) {
|
|
3878
|
+
cuda_backend = ggml_backend_cuda_init(i);
|
|
3879
|
+
break;
|
|
3880
|
+
}
|
|
3881
|
+
}
|
|
3882
|
+
}
|
|
3883
|
+
|
|
3884
|
+
// If the cuda backend is active create pinned memory buffers and events for synchronisation.
|
|
3885
|
+
if (cuda_backend) {
|
|
3886
|
+
for (size_t idx = 0; idx < num_buffers; ++idx) {
|
|
3887
|
+
host_buffers.emplace_back(ggml_backend_buft_alloc_buffer(llama_default_buffer_type_cpu(true), buffer_size));
|
|
3888
|
+
host_ptrs.emplace_back(ggml_backend_buffer_get_base(host_buffers[idx]));
|
|
3889
|
+
events.emplace_back(ggml_backend_event_new(cuda_backend));
|
|
3890
|
+
}
|
|
3891
|
+
}
|
|
3892
|
+
}
|
|
3893
|
+
#endif
|
|
3894
|
+
|
|
3715
3895
|
for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur != NULL; cur = ggml_get_next_tensor(ctx, cur)) {
|
|
3716
3896
|
const auto * weight = get_weight(ggml_get_name(cur));
|
|
3717
3897
|
if (weight == nullptr) {
|
|
@@ -3767,12 +3947,36 @@ struct llama_model_loader {
|
|
|
3767
3947
|
}));
|
|
3768
3948
|
}
|
|
3769
3949
|
} else {
|
|
3770
|
-
|
|
3771
|
-
|
|
3772
|
-
|
|
3773
|
-
|
|
3774
|
-
|
|
3775
|
-
|
|
3950
|
+
#if defined(GGML_USE_CUDA)
|
|
3951
|
+
// If cuda_backend is valid load the tensor in chunks to pinned memory and upload the buffers asynchronously to the GPU.
|
|
3952
|
+
if (cuda_backend) {
|
|
3953
|
+
file->seek(weight->offs, SEEK_SET);
|
|
3954
|
+
|
|
3955
|
+
size_t bytes_read = 0;
|
|
3956
|
+
|
|
3957
|
+
while (bytes_read < n_size) {
|
|
3958
|
+
size_t read_iteration = std::min<size_t>(buffer_size, n_size - bytes_read);
|
|
3959
|
+
|
|
3960
|
+
ggml_backend_event_synchronize(events[buffer_idx]);
|
|
3961
|
+
file->read_raw(host_ptrs[buffer_idx], read_iteration);
|
|
3962
|
+
ggml_backend_tensor_set_async(cuda_backend, cur, host_ptrs[buffer_idx], bytes_read, read_iteration);
|
|
3963
|
+
ggml_backend_event_record(events[buffer_idx]);
|
|
3964
|
+
|
|
3965
|
+
bytes_read += read_iteration;
|
|
3966
|
+
++buffer_idx;
|
|
3967
|
+
buffer_idx %= num_buffers;
|
|
3968
|
+
}
|
|
3969
|
+
}
|
|
3970
|
+
else
|
|
3971
|
+
#endif
|
|
3972
|
+
{
|
|
3973
|
+
read_buf.resize(n_size);
|
|
3974
|
+
file->seek(weight->offs, SEEK_SET);
|
|
3975
|
+
file->read_raw(read_buf.data(), n_size);
|
|
3976
|
+
ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size);
|
|
3977
|
+
if (check_tensors && !ggml_validate_row_data(cur->type, read_buf.data(), n_size)) {
|
|
3978
|
+
throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));
|
|
3979
|
+
}
|
|
3776
3980
|
}
|
|
3777
3981
|
}
|
|
3778
3982
|
}
|
|
@@ -3780,6 +3984,18 @@ struct llama_model_loader {
|
|
|
3780
3984
|
size_done += n_size;
|
|
3781
3985
|
}
|
|
3782
3986
|
|
|
3987
|
+
#if defined(GGML_USE_CUDA)
|
|
3988
|
+
// free temporary resources used for async cuda uploads
|
|
3989
|
+
if (cuda_backend) {
|
|
3990
|
+
for (size_t idx = 0; idx < num_buffers;++idx) {
|
|
3991
|
+
ggml_backend_event_synchronize(events[idx]);
|
|
3992
|
+
ggml_backend_event_free(events[idx]);
|
|
3993
|
+
ggml_backend_buffer_free(host_buffers[idx]);
|
|
3994
|
+
}
|
|
3995
|
+
ggml_backend_free(cuda_backend);
|
|
3996
|
+
}
|
|
3997
|
+
#endif
|
|
3998
|
+
|
|
3783
3999
|
// check validation results
|
|
3784
4000
|
bool validation_failed = false;
|
|
3785
4001
|
for (auto & future : validation_result) {
|
|
@@ -4246,6 +4462,9 @@ static void llm_load_hparams(
|
|
|
4246
4462
|
} break;
|
|
4247
4463
|
case LLM_ARCH_QWEN2MOE:
|
|
4248
4464
|
{
|
|
4465
|
+
ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);
|
|
4466
|
+
ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);
|
|
4467
|
+
|
|
4249
4468
|
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
|
4250
4469
|
switch (hparams.n_layer) {
|
|
4251
4470
|
case 24: model.type = e_model::MODEL_A2_7B; break;
|
|
@@ -4552,38 +4771,9 @@ static void llm_load_vocab(
|
|
|
4552
4771
|
vocab.special_cls_id = -1;
|
|
4553
4772
|
vocab.special_mask_id = -1;
|
|
4554
4773
|
|
|
4555
|
-
// For Fill-In-the-Middle (FIM)/infill models which where converted
|
|
4556
|
-
// prior to support of FIM special tokens in GGUF, the following
|
|
4557
|
-
// will allow those models to continue to work. The general names
|
|
4558
|
-
// of the known models are currently CodeLlama (LLM_ARCH_LLAMA) and
|
|
4559
|
-
// CodeGemma (LLM_ARCH_GEMMA). This can potentially be removed once
|
|
4560
|
-
// new versions of these models have been published.
|
|
4561
|
-
std::string gen_name;
|
|
4562
|
-
ml.get_key(LLM_KV_GENERAL_NAME, gen_name, false);
|
|
4563
|
-
|
|
4564
|
-
std::transform(gen_name.begin(), gen_name.end(), gen_name.begin(),
|
|
4565
|
-
[](unsigned char c){ return std::tolower(c); });
|
|
4566
|
-
|
|
4567
|
-
if (gen_name.find("code") != std::string::npos) {
|
|
4568
|
-
if (model.arch == LLM_ARCH_LLAMA) {
|
|
4569
|
-
vocab.special_prefix_id = 32007;
|
|
4570
|
-
vocab.special_suffix_id = 32008;
|
|
4571
|
-
vocab.special_middle_id = 32009;
|
|
4572
|
-
vocab.special_eot_id = 32010;
|
|
4573
|
-
} else if (model.arch == LLM_ARCH_GEMMA) {
|
|
4574
|
-
vocab.special_prefix_id = 67;
|
|
4575
|
-
vocab.special_suffix_id = 69;
|
|
4576
|
-
vocab.special_middle_id = 68;
|
|
4577
|
-
// TODO: this is not EOT, it is "file separator" token, needs fix
|
|
4578
|
-
// https://huggingface.co/google/codegemma-7b-it/blob/9b1d9231388358c04d90bd003458f5070d97db44/tokenizer_config.json#L565-L572
|
|
4579
|
-
//vocab.special_eot_id = 70;
|
|
4580
|
-
vocab.special_eot_id = 107;
|
|
4581
|
-
}
|
|
4582
|
-
}
|
|
4583
|
-
|
|
4584
4774
|
const int add_space_prefix_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_ADD_PREFIX).c_str());
|
|
4585
4775
|
if (add_space_prefix_keyidx != -1) {
|
|
4586
|
-
vocab.
|
|
4776
|
+
vocab.tokenizer_add_space_prefix = gguf_get_val_bool(ctx, add_space_prefix_keyidx);
|
|
4587
4777
|
} // The default value of add_space_prefix is true.
|
|
4588
4778
|
} else if (tokenizer_model == "bert") {
|
|
4589
4779
|
vocab.type = LLAMA_VOCAB_TYPE_WPM;
|
|
@@ -4596,13 +4786,13 @@ static void llm_load_vocab(
|
|
|
4596
4786
|
vocab.special_pad_id = 0;
|
|
4597
4787
|
vocab.special_cls_id = 101;
|
|
4598
4788
|
vocab.special_mask_id = 103;
|
|
4599
|
-
vocab.
|
|
4789
|
+
vocab.tokenizer_add_space_prefix = false;
|
|
4600
4790
|
} else if (tokenizer_model == "gpt2") {
|
|
4601
4791
|
vocab.type = LLAMA_VOCAB_TYPE_BPE;
|
|
4602
4792
|
|
|
4603
4793
|
const int add_space_prefix_keyidx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_ADD_PREFIX).c_str());
|
|
4604
4794
|
if (add_space_prefix_keyidx != -1) {
|
|
4605
|
-
vocab.
|
|
4795
|
+
vocab.tokenizer_add_space_prefix = gguf_get_val_bool(ctx, add_space_prefix_keyidx);
|
|
4606
4796
|
}
|
|
4607
4797
|
|
|
4608
4798
|
// read bpe merges and populate bpe ranks
|
|
@@ -4653,14 +4843,15 @@ static void llm_load_vocab(
|
|
|
4653
4843
|
LLAMA_LOG_WARN("%s: ************************************ \n", __func__);
|
|
4654
4844
|
LLAMA_LOG_WARN("%s: \n", __func__);
|
|
4655
4845
|
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
|
|
4656
|
-
} else if (
|
|
4657
|
-
tokenizer_pre == "default") {
|
|
4846
|
+
} else if (tokenizer_pre == "default") {
|
|
4658
4847
|
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
|
|
4659
4848
|
} else if (
|
|
4660
4849
|
tokenizer_pre == "llama3" ||
|
|
4661
4850
|
tokenizer_pre == "llama-v3" ||
|
|
4662
4851
|
tokenizer_pre == "llama-bpe") {
|
|
4663
4852
|
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_LLAMA3;
|
|
4853
|
+
vocab.tokenizer_ignore_merges = true;
|
|
4854
|
+
vocab.tokenizer_add_bos = true;
|
|
4664
4855
|
} else if (
|
|
4665
4856
|
tokenizer_pre == "deepseek-llm") {
|
|
4666
4857
|
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM;
|
|
@@ -4681,7 +4872,8 @@ static void llm_load_vocab(
|
|
|
4681
4872
|
tokenizer_pre == "jina-es" ||
|
|
4682
4873
|
tokenizer_pre == "jina-de" ||
|
|
4683
4874
|
tokenizer_pre == "jina-v2-es" ||
|
|
4684
|
-
tokenizer_pre == "jina-v2-de"
|
|
4875
|
+
tokenizer_pre == "jina-v2-de" ||
|
|
4876
|
+
tokenizer_pre == "jina-v2-code") {
|
|
4685
4877
|
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2;
|
|
4686
4878
|
} else if (
|
|
4687
4879
|
tokenizer_pre == "refact") {
|
|
@@ -4704,9 +4896,20 @@ static void llm_load_vocab(
|
|
|
4704
4896
|
} else if (
|
|
4705
4897
|
tokenizer_pre == "smaug-bpe") {
|
|
4706
4898
|
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMAUG;
|
|
4899
|
+
} else if (
|
|
4900
|
+
tokenizer_pre == "poro-chat") {
|
|
4901
|
+
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_PORO;
|
|
4707
4902
|
} else {
|
|
4708
4903
|
throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
|
|
4709
4904
|
}
|
|
4905
|
+
} else if (vocab.type == LLAMA_VOCAB_TYPE_SPM) {
|
|
4906
|
+
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
|
|
4907
|
+
vocab.tokenizer_add_bos = true;
|
|
4908
|
+
vocab.tokenizer_add_eos = false;
|
|
4909
|
+
} else if (vocab.type == LLAMA_VOCAB_TYPE_WPM) {
|
|
4910
|
+
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
|
|
4911
|
+
vocab.tokenizer_add_bos = true;
|
|
4912
|
+
vocab.tokenizer_add_eos = false;
|
|
4710
4913
|
} else {
|
|
4711
4914
|
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
|
|
4712
4915
|
}
|
|
@@ -4738,6 +4941,7 @@ static void llm_load_vocab(
|
|
|
4738
4941
|
GGML_ASSERT(unicode_cpts_from_utf8(word).size() > 0);
|
|
4739
4942
|
|
|
4740
4943
|
vocab.token_to_id[word] = i;
|
|
4944
|
+
vocab.max_token_len = std::max(vocab.max_token_len, (int) word.size());
|
|
4741
4945
|
|
|
4742
4946
|
auto & token_data = vocab.id_to_token[i];
|
|
4743
4947
|
token_data.text = std::move(word);
|
|
@@ -4761,6 +4965,45 @@ static void llm_load_vocab(
|
|
|
4761
4965
|
|
|
4762
4966
|
// determine the newline token: LLaMA "<0x0A>" == 10 == '\n', Falcon 193 == '\n'
|
|
4763
4967
|
if (vocab.type == LLAMA_VOCAB_TYPE_SPM) {
|
|
4968
|
+
// For Fill-In-the-Middle (FIM)/infill models which where converted
|
|
4969
|
+
// prior to support of FIM special tokens in GGUF, the following
|
|
4970
|
+
// will allow those models to continue to work. The general names
|
|
4971
|
+
// of the known models are currently CodeLlama (LLM_ARCH_LLAMA) and
|
|
4972
|
+
// CodeGemma (LLM_ARCH_GEMMA). This can potentially be removed once
|
|
4973
|
+
// new versions of these models have been published.
|
|
4974
|
+
std::string gen_name;
|
|
4975
|
+
ml.get_key(LLM_KV_GENERAL_NAME, gen_name, false);
|
|
4976
|
+
|
|
4977
|
+
std::transform(gen_name.begin(), gen_name.end(), gen_name.begin(),
|
|
4978
|
+
[](unsigned char c){ return std::tolower(c); });
|
|
4979
|
+
|
|
4980
|
+
if (gen_name.find("code") != std::string::npos) {
|
|
4981
|
+
if (model.arch == LLM_ARCH_LLAMA
|
|
4982
|
+
&& 32010 < vocab.id_to_token.size()
|
|
4983
|
+
&& vocab.id_to_token[32007].text == "<PRE>"
|
|
4984
|
+
&& vocab.id_to_token[32008].text == "<SUF>"
|
|
4985
|
+
&& vocab.id_to_token[32009].text == "<MID>"
|
|
4986
|
+
&& vocab.id_to_token[32010].text == "<EOT>") {
|
|
4987
|
+
vocab.special_prefix_id = 32007;
|
|
4988
|
+
vocab.special_suffix_id = 32008;
|
|
4989
|
+
vocab.special_middle_id = 32009;
|
|
4990
|
+
vocab.special_eot_id = 32010;
|
|
4991
|
+
} else if (model.arch == LLM_ARCH_GEMMA
|
|
4992
|
+
&& 107 < vocab.id_to_token.size()
|
|
4993
|
+
&& vocab.id_to_token[67].text == "<|fim_prefix|>"
|
|
4994
|
+
&& vocab.id_to_token[69].text == "<|fim_suffix|>"
|
|
4995
|
+
&& vocab.id_to_token[68].text == "<|fim_middle|>"
|
|
4996
|
+
&& vocab.id_to_token[107].text == "<end_of_turn>") {
|
|
4997
|
+
vocab.special_prefix_id = 67;
|
|
4998
|
+
vocab.special_suffix_id = 69;
|
|
4999
|
+
vocab.special_middle_id = 68;
|
|
5000
|
+
// TODO: this is not EOT, it is "file separator" token, needs fix
|
|
5001
|
+
// https://huggingface.co/google/codegemma-7b-it/blob/9b1d9231388358c04d90bd003458f5070d97db44/tokenizer_config.json#L565-L572
|
|
5002
|
+
//vocab.special_eot_id = 70;
|
|
5003
|
+
vocab.special_eot_id = 107;
|
|
5004
|
+
}
|
|
5005
|
+
}
|
|
5006
|
+
|
|
4764
5007
|
try {
|
|
4765
5008
|
vocab.linefeed_id = llama_byte_to_token(vocab, '\n');
|
|
4766
5009
|
} catch (const std::exception & e) {
|
|
@@ -4812,10 +5055,10 @@ static void llm_load_vocab(
|
|
|
4812
5055
|
bool temp = true;
|
|
4813
5056
|
|
|
4814
5057
|
if (ml.get_key(LLM_KV_TOKENIZER_ADD_BOS, temp, false)) {
|
|
4815
|
-
vocab.
|
|
5058
|
+
vocab.tokenizer_add_bos = temp;
|
|
4816
5059
|
}
|
|
4817
5060
|
if (ml.get_key(LLM_KV_TOKENIZER_ADD_EOS, temp, false)) {
|
|
4818
|
-
vocab.
|
|
5061
|
+
vocab.tokenizer_add_eos = temp;
|
|
4819
5062
|
}
|
|
4820
5063
|
}
|
|
4821
5064
|
|
|
@@ -4915,7 +5158,7 @@ static void llm_load_vocab(
|
|
|
4915
5158
|
);
|
|
4916
5159
|
|
|
4917
5160
|
// set attributes by model/tokenizer name
|
|
4918
|
-
if (_contains_any(tokenizer_pre, {"jina-v2-es", "jina-v2-
|
|
5161
|
+
if (_contains_any(tokenizer_pre, {"jina-v2-de", "jina-v2-es", "jina-v2-code"})) {
|
|
4919
5162
|
_set_token_attr("<mask>", LLAMA_TOKEN_ATTR_LSTRIP, true);
|
|
4920
5163
|
} else if (_contains_any(model_name, {"phi-3", "phi3"})) {
|
|
4921
5164
|
for (auto id : vocab.cache_special_tokens) {
|
|
@@ -5009,6 +5252,8 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {
|
|
|
5009
5252
|
if (vocab.special_middle_id != -1) { LLAMA_LOG_INFO( "%s: MID token = %d '%s'\n", __func__, vocab.special_middle_id, vocab.id_to_token[vocab.special_middle_id].text.c_str() ); }
|
|
5010
5253
|
if (vocab.special_eot_id != -1) { LLAMA_LOG_INFO( "%s: EOT token = %d '%s'\n", __func__, vocab.special_eot_id, vocab.id_to_token[vocab.special_eot_id].text.c_str() ); }
|
|
5011
5254
|
|
|
5255
|
+
LLAMA_LOG_INFO("%s: max token length = %d\n", __func__, vocab.max_token_len);
|
|
5256
|
+
|
|
5012
5257
|
if (model.arch == LLM_ARCH_DEEPSEEK2) {
|
|
5013
5258
|
LLAMA_LOG_INFO("%s: n_layer_dense_lead = %d\n", __func__, hparams.n_layer_dense_lead);
|
|
5014
5259
|
LLAMA_LOG_INFO("%s: n_lora_q = %d\n", __func__, hparams.n_lora_q);
|
|
@@ -5018,6 +5263,11 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {
|
|
|
5018
5263
|
LLAMA_LOG_INFO("%s: expert_weights_scale = %.1f\n", __func__, hparams.expert_weights_scale);
|
|
5019
5264
|
LLAMA_LOG_INFO("%s: rope_yarn_log_mul = %.4f\n", __func__, hparams.rope_yarn_log_mul);
|
|
5020
5265
|
}
|
|
5266
|
+
|
|
5267
|
+
if (model.arch == LLM_ARCH_QWEN2MOE) {
|
|
5268
|
+
LLAMA_LOG_INFO("%s: n_ff_exp = %d\n", __func__, hparams.n_ff_exp);
|
|
5269
|
+
LLAMA_LOG_INFO("%s: n_ff_shexp = %d\n", __func__, hparams.n_ff_shexp);
|
|
5270
|
+
}
|
|
5021
5271
|
}
|
|
5022
5272
|
|
|
5023
5273
|
// Returns false if cancelled by progress_callback
|
|
@@ -5161,7 +5411,7 @@ static bool llm_load_tensors(
|
|
|
5161
5411
|
// create tensors for the weights
|
|
5162
5412
|
{
|
|
5163
5413
|
const int64_t n_embd = hparams.n_embd;
|
|
5164
|
-
const int64_t n_embd_head = n_embd / hparams.n_head;
|
|
5414
|
+
const int64_t n_embd_head = (hparams.n_head == 0) ? 0 : n_embd / hparams.n_head;
|
|
5165
5415
|
const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa();
|
|
5166
5416
|
const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa();
|
|
5167
5417
|
const int64_t n_embd_gqa = n_embd_v_gqa;
|
|
@@ -5515,7 +5765,7 @@ static bool llm_load_tensors(
|
|
|
5515
5765
|
|
|
5516
5766
|
layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd});
|
|
5517
5767
|
} else {
|
|
5518
|
-
layer.ffn_gate
|
|
5768
|
+
layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
|
|
5519
5769
|
}
|
|
5520
5770
|
|
|
5521
5771
|
layer.layer_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd});
|
|
@@ -5556,6 +5806,9 @@ static bool llm_load_tensors(
|
|
|
5556
5806
|
layer.attn_out_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}); //output_norm
|
|
5557
5807
|
layer.attn_out_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd});
|
|
5558
5808
|
|
|
5809
|
+
layer.attn_norm_2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
|
|
5810
|
+
layer.attn_norm_2_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
|
|
5811
|
+
|
|
5559
5812
|
layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
|
|
5560
5813
|
layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
|
|
5561
5814
|
|
|
@@ -5801,16 +6054,17 @@ static bool llm_load_tensors(
|
|
|
5801
6054
|
GGML_ASSERT(hparams.n_expert_used > 0);
|
|
5802
6055
|
|
|
5803
6056
|
// MoE branch
|
|
5804
|
-
auto n_ff_exp = n_ff / hparams.n_expert_used;
|
|
6057
|
+
auto n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / hparams.n_expert_used;
|
|
5805
6058
|
layer.ffn_gate_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert});
|
|
5806
6059
|
layer.ffn_down_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert});
|
|
5807
6060
|
layer.ffn_up_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert});
|
|
5808
6061
|
|
|
5809
6062
|
// Shared expert branch
|
|
6063
|
+
auto n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff;
|
|
5810
6064
|
layer.ffn_gate_inp_shexp = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", i), {n_embd});
|
|
5811
|
-
layer.ffn_gate_shexp = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd,
|
|
5812
|
-
layer.ffn_down_shexp = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {
|
|
5813
|
-
layer.ffn_up_shexp = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd,
|
|
6065
|
+
layer.ffn_gate_shexp = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp});
|
|
6066
|
+
layer.ffn_down_shexp = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd});
|
|
6067
|
+
layer.ffn_up_shexp = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp});
|
|
5814
6068
|
}
|
|
5815
6069
|
} break;
|
|
5816
6070
|
case LLM_ARCH_PHI2:
|
|
@@ -6600,16 +6854,6 @@ static int llama_model_load(const std::string & fname, llama_model & model, llam
|
|
|
6600
6854
|
}
|
|
6601
6855
|
#endif
|
|
6602
6856
|
|
|
6603
|
-
#ifdef GGML_USE_SYCL
|
|
6604
|
-
if (params.split_mode == LLAMA_SPLIT_MODE_NONE) {
|
|
6605
|
-
ggml_backend_sycl_set_single_device_mode(params.main_gpu);
|
|
6606
|
-
//SYCL use device index (0, 1, 2) directly, uer input device id, then convert to device index.
|
|
6607
|
-
params.main_gpu = ggml_backend_sycl_get_device_index(params.main_gpu);
|
|
6608
|
-
} else {
|
|
6609
|
-
ggml_backend_sycl_set_mul_device_mode();
|
|
6610
|
-
}
|
|
6611
|
-
#endif
|
|
6612
|
-
|
|
6613
6857
|
if (!llm_load_tensors(
|
|
6614
6858
|
ml, model, params.n_gpu_layers, params.split_mode, params.main_gpu, params.tensor_split, params.use_mlock,
|
|
6615
6859
|
params.progress_callback, params.progress_callback_user_data
|
|
@@ -7410,6 +7654,50 @@ struct llm_build_context {
|
|
|
7410
7654
|
return lctx.inp_s_seq;
|
|
7411
7655
|
}
|
|
7412
7656
|
|
|
7657
|
+
struct ggml_cgraph * append_pooling(struct ggml_cgraph * gf) {
|
|
7658
|
+
// find result_norm tensor for input
|
|
7659
|
+
struct ggml_tensor * inp = nullptr;
|
|
7660
|
+
for (int i = gf->n_nodes - 1; i >= 0; --i) {
|
|
7661
|
+
inp = gf->nodes[i];
|
|
7662
|
+
if (strcmp(inp->name, "result_norm") == 0 || strcmp(inp->name, "result_embd") == 0) {
|
|
7663
|
+
break;
|
|
7664
|
+
} else {
|
|
7665
|
+
inp = nullptr;
|
|
7666
|
+
}
|
|
7667
|
+
}
|
|
7668
|
+
GGML_ASSERT(inp != nullptr && "missing result_norm/result_embd tensor");
|
|
7669
|
+
|
|
7670
|
+
struct ggml_tensor * cur;
|
|
7671
|
+
|
|
7672
|
+
switch (pooling_type) {
|
|
7673
|
+
case LLAMA_POOLING_TYPE_MEAN:
|
|
7674
|
+
{
|
|
7675
|
+
struct ggml_tensor * inp_mean = build_inp_mean();
|
|
7676
|
+
cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, inp)), inp_mean);
|
|
7677
|
+
} break;
|
|
7678
|
+
case LLAMA_POOLING_TYPE_CLS:
|
|
7679
|
+
case LLAMA_POOLING_TYPE_LAST:
|
|
7680
|
+
{
|
|
7681
|
+
struct ggml_tensor * inp_cls = build_inp_cls();
|
|
7682
|
+
cur = ggml_get_rows(ctx0, inp, inp_cls);
|
|
7683
|
+
} break;
|
|
7684
|
+
case LLAMA_POOLING_TYPE_NONE:
|
|
7685
|
+
{
|
|
7686
|
+
cur = inp;
|
|
7687
|
+
} break;
|
|
7688
|
+
default:
|
|
7689
|
+
{
|
|
7690
|
+
GGML_ASSERT(false && "unknown pooling type");
|
|
7691
|
+
} break;
|
|
7692
|
+
}
|
|
7693
|
+
|
|
7694
|
+
cb(cur, "result_embd_pooled", -1);
|
|
7695
|
+
|
|
7696
|
+
ggml_build_forward_expand(gf, cur);
|
|
7697
|
+
|
|
7698
|
+
return gf;
|
|
7699
|
+
}
|
|
7700
|
+
|
|
7413
7701
|
struct ggml_cgraph * build_llama() {
|
|
7414
7702
|
struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);
|
|
7415
7703
|
|
|
@@ -8390,8 +8678,6 @@ struct llm_build_context {
|
|
|
8390
8678
|
if (model.arch != LLM_ARCH_JINA_BERT_V2) {
|
|
8391
8679
|
inp_pos = build_inp_pos();
|
|
8392
8680
|
}
|
|
8393
|
-
struct ggml_tensor * inp_mean = build_inp_mean();
|
|
8394
|
-
struct ggml_tensor * inp_cls = build_inp_cls();
|
|
8395
8681
|
|
|
8396
8682
|
// construct input embeddings (token, type, position)
|
|
8397
8683
|
inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
|
|
@@ -8519,6 +8805,11 @@ struct llm_build_context {
|
|
|
8519
8805
|
// attention layer norm
|
|
8520
8806
|
cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_out_norm, model.layers[il].attn_out_norm_b, LLM_NORM, cb, il);
|
|
8521
8807
|
|
|
8808
|
+
if (model.layers[il].attn_norm_2 != nullptr) {
|
|
8809
|
+
cur = ggml_add(ctx0, cur, inpL); // re-add the layer input
|
|
8810
|
+
cur = llm_build_norm(ctx0, cur, hparams, model.layers[il].attn_norm_2, model.layers[il].attn_norm_2_b, LLM_NORM, cb, il);
|
|
8811
|
+
}
|
|
8812
|
+
|
|
8522
8813
|
struct ggml_tensor * ffn_inp = cur;
|
|
8523
8814
|
cb(ffn_inp, "ffn_inp", il);
|
|
8524
8815
|
|
|
@@ -8561,28 +8852,6 @@ struct llm_build_context {
|
|
|
8561
8852
|
cur = inpL;
|
|
8562
8853
|
cb(cur, "result_embd", -1);
|
|
8563
8854
|
|
|
8564
|
-
// pooling layer
|
|
8565
|
-
switch (pooling_type) {
|
|
8566
|
-
case LLAMA_POOLING_TYPE_NONE:
|
|
8567
|
-
{
|
|
8568
|
-
// nop
|
|
8569
|
-
} break;
|
|
8570
|
-
case LLAMA_POOLING_TYPE_MEAN:
|
|
8571
|
-
{
|
|
8572
|
-
cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, cur)), inp_mean);
|
|
8573
|
-
cb(cur, "result_embd_pooled", -1);
|
|
8574
|
-
} break;
|
|
8575
|
-
case LLAMA_POOLING_TYPE_CLS:
|
|
8576
|
-
{
|
|
8577
|
-
cur = ggml_get_rows(ctx0, cur, inp_cls);
|
|
8578
|
-
cb(cur, "result_embd_pooled", -1);
|
|
8579
|
-
} break;
|
|
8580
|
-
case LLAMA_POOLING_TYPE_UNSPECIFIED:
|
|
8581
|
-
{
|
|
8582
|
-
GGML_ASSERT(false && "Invalid pooling type");
|
|
8583
|
-
} break;
|
|
8584
|
-
}
|
|
8585
|
-
|
|
8586
8855
|
ggml_build_forward_expand(gf, cur);
|
|
8587
8856
|
|
|
8588
8857
|
return gf;
|
|
@@ -11520,7 +11789,8 @@ static struct ggml_cgraph * llama_build_graph(
|
|
|
11520
11789
|
if (batch.n_tokens < 32 || full_offload) {
|
|
11521
11790
|
if (il != -1 && strcmp(name, "norm") == 0) {
|
|
11522
11791
|
for (auto * backend : lctx.backends) {
|
|
11523
|
-
if (
|
|
11792
|
+
if (ggml_backend_supports_buft(backend, lctx.model.buft_layer[il].buft) &&
|
|
11793
|
+
(ggml_backend_supports_op(backend, cur) || ggml_backend_offload_op(backend, cur))) {
|
|
11524
11794
|
ggml_backend_sched_set_tensor_backend(lctx.sched, cur, backend);
|
|
11525
11795
|
break;
|
|
11526
11796
|
}
|
|
@@ -11666,6 +11936,11 @@ static struct ggml_cgraph * llama_build_graph(
|
|
|
11666
11936
|
GGML_ASSERT(false);
|
|
11667
11937
|
}
|
|
11668
11938
|
|
|
11939
|
+
// add on pooling layer
|
|
11940
|
+
if (lctx.cparams.embeddings) {
|
|
11941
|
+
result = llm.append_pooling(result);
|
|
11942
|
+
}
|
|
11943
|
+
|
|
11669
11944
|
llm.free();
|
|
11670
11945
|
|
|
11671
11946
|
return result;
|
|
@@ -11755,7 +12030,7 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) {
|
|
|
11755
12030
|
// (!a || b) is a logical implication (a -> b)
|
|
11756
12031
|
// !hparams.causal_attn -> !cparams.causal_attn
|
|
11757
12032
|
(hparams.causal_attn || !cparams.causal_attn) &&
|
|
11758
|
-
"causal attention
|
|
12033
|
+
"causal attention is not supported by this model"
|
|
11759
12034
|
);
|
|
11760
12035
|
|
|
11761
12036
|
if (lctx.inp_KQ_mask) {
|
|
@@ -11887,6 +12162,37 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) {
|
|
|
11887
12162
|
}
|
|
11888
12163
|
}
|
|
11889
12164
|
|
|
12165
|
+
if (cparams.pooling_type == LLAMA_POOLING_TYPE_LAST) {
|
|
12166
|
+
const int64_t n_tokens = batch.n_tokens;
|
|
12167
|
+
|
|
12168
|
+
GGML_ASSERT(lctx.inp_cls);
|
|
12169
|
+
GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_cls->buffer));
|
|
12170
|
+
|
|
12171
|
+
uint32_t * data = (uint32_t *) lctx.inp_cls->data;
|
|
12172
|
+
memset(lctx.inp_cls->data, 0, n_tokens * ggml_element_size(lctx.inp_cls));
|
|
12173
|
+
|
|
12174
|
+
std::vector<int> last_pos(n_tokens, -1);
|
|
12175
|
+
std::vector<int> last_row(n_tokens, -1);
|
|
12176
|
+
|
|
12177
|
+
for (int i = 0; i < n_tokens; ++i) {
|
|
12178
|
+
const llama_seq_id seq_id = batch.seq_id[i][0];
|
|
12179
|
+
const llama_pos pos = batch.pos[i];
|
|
12180
|
+
|
|
12181
|
+
GGML_ASSERT(seq_id < n_tokens && "seq_id cannot be larger than n_tokens with pooling_type == LAST");
|
|
12182
|
+
|
|
12183
|
+
if (pos >= last_pos[seq_id]) {
|
|
12184
|
+
last_pos[seq_id] = pos;
|
|
12185
|
+
last_row[seq_id] = i;
|
|
12186
|
+
}
|
|
12187
|
+
}
|
|
12188
|
+
|
|
12189
|
+
for (int i = 0; i < n_tokens; ++i) {
|
|
12190
|
+
if (last_row[i] >= 0) {
|
|
12191
|
+
data[i] = last_row[i];
|
|
12192
|
+
}
|
|
12193
|
+
}
|
|
12194
|
+
}
|
|
12195
|
+
|
|
11890
12196
|
if (kv_self.recurrent) {
|
|
11891
12197
|
const int64_t n_kv = kv_self.n;
|
|
11892
12198
|
|
|
@@ -11948,8 +12254,8 @@ static size_t llama_output_reserve(llama_context & lctx, size_t n_outputs) {
|
|
|
11948
12254
|
const auto n_embd = hparams.n_embd;
|
|
11949
12255
|
|
|
11950
12256
|
// TODO: use a per-batch flag for logits presence instead
|
|
11951
|
-
const bool has_logits = cparams.
|
|
11952
|
-
const bool has_embd =
|
|
12257
|
+
const bool has_logits = !cparams.embeddings;
|
|
12258
|
+
const bool has_embd = cparams.embeddings && (cparams.pooling_type == LLAMA_POOLING_TYPE_NONE);
|
|
11953
12259
|
|
|
11954
12260
|
const size_t logits_size = has_logits ? n_vocab*n_outputs_max : 0;
|
|
11955
12261
|
const size_t embd_size = has_embd ? n_embd*n_outputs_max : 0;
|
|
@@ -12017,6 +12323,11 @@ static void llama_graph_compute(
|
|
|
12017
12323
|
ggml_backend_cpu_set_n_threads(lctx.backend_cpu, n_threads);
|
|
12018
12324
|
ggml_backend_cpu_set_abort_callback(lctx.backend_cpu, lctx.abort_callback, lctx.abort_callback_data);
|
|
12019
12325
|
}
|
|
12326
|
+
#ifdef GGML_USE_BLAS
|
|
12327
|
+
if (lctx.backend_blas != nullptr) {
|
|
12328
|
+
ggml_backend_blas_set_n_threads(lctx.backend_blas, n_threads);
|
|
12329
|
+
}
|
|
12330
|
+
#endif
|
|
12020
12331
|
|
|
12021
12332
|
ggml_backend_sched_graph_compute_async(lctx.sched, gf);
|
|
12022
12333
|
|
|
@@ -12074,11 +12385,13 @@ static int llama_decode_internal(
|
|
|
12074
12385
|
std::vector<std::vector<llama_seq_id>> seq_id;
|
|
12075
12386
|
|
|
12076
12387
|
// count outputs
|
|
12077
|
-
if (
|
|
12388
|
+
if (cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE) {
|
|
12389
|
+
n_outputs = n_tokens_all;
|
|
12390
|
+
} else if (batch_all.logits) {
|
|
12078
12391
|
for (uint32_t i = 0; i < n_tokens_all; ++i) {
|
|
12079
12392
|
n_outputs += batch_all.logits[i] != 0;
|
|
12080
12393
|
}
|
|
12081
|
-
} else if (lctx.logits_all
|
|
12394
|
+
} else if (lctx.logits_all) {
|
|
12082
12395
|
n_outputs = n_tokens_all;
|
|
12083
12396
|
} else {
|
|
12084
12397
|
// keep last output only
|
|
@@ -12209,47 +12522,19 @@ static int llama_decode_internal(
|
|
|
12209
12522
|
// no output
|
|
12210
12523
|
res = nullptr;
|
|
12211
12524
|
embd = nullptr;
|
|
12212
|
-
} else if (!hparams.causal_attn) {
|
|
12213
|
-
res = nullptr; // do not extract logits for embedding models such as BERT
|
|
12214
|
-
|
|
12215
|
-
// token or sequence embeddings
|
|
12216
|
-
embd = gf->nodes[gf->n_nodes - 1];
|
|
12217
|
-
|
|
12218
|
-
GGML_ASSERT(strcmp(embd->name, "result_embd") == 0 || strcmp(embd->name, "result_embd_pooled") == 0);
|
|
12219
12525
|
} else if (cparams.embeddings) {
|
|
12220
|
-
|
|
12221
|
-
|
|
12222
|
-
|
|
12223
|
-
|
|
12224
|
-
if (i_embd < 0) { break; }
|
|
12225
|
-
embd = gf->nodes[i_embd];
|
|
12226
|
-
}
|
|
12227
|
-
GGML_ASSERT(i_embd >= 0 && "missing result_norm tensor");
|
|
12228
|
-
|
|
12229
|
-
// TODO: use a per-batch flag to know when to skip logits while keeping embeddings
|
|
12230
|
-
if (!cparams.causal_attn) {
|
|
12231
|
-
res = nullptr; // do not extract logits when not needed
|
|
12232
|
-
// skip computing logits
|
|
12233
|
-
// TODO: is this safe?
|
|
12234
|
-
gf->n_nodes = i_embd + 1;
|
|
12526
|
+
res = nullptr; // do not extract logits for embedding case
|
|
12527
|
+
embd = gf->nodes[gf->n_nodes - 1];
|
|
12528
|
+
if (strcmp(embd->name, "result_embd_pooled") != 0) {
|
|
12529
|
+
embd = gf->nodes[gf->n_nodes - 2];
|
|
12235
12530
|
}
|
|
12531
|
+
GGML_ASSERT(strcmp(embd->name, "result_embd_pooled") == 0 && "missing embeddings tensor");
|
|
12236
12532
|
} else {
|
|
12237
12533
|
embd = nullptr; // do not extract embeddings when not needed
|
|
12238
12534
|
GGML_ASSERT(strcmp(res->name, "result_output") == 0 && "missing result_output tensor");
|
|
12239
12535
|
}
|
|
12240
12536
|
// LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs);
|
|
12241
12537
|
|
|
12242
|
-
// for big prompts, if BLAS is enabled, it is better to use only one thread
|
|
12243
|
-
// otherwise, the threads are spin-lock waiting for the BLAS calls and are degrading the performance
|
|
12244
|
-
// TODO: this is mostly important for Apple Silicon where CBLAS is still performing very well
|
|
12245
|
-
// we still need some threads to process all non-mul_mat ops, but not too much to avoid interfering
|
|
12246
|
-
// with the BLAS calls. need a better solution
|
|
12247
|
-
// MoE Special Case: This logic applies when hparams.n_expert == 0, i.e. the model is NOT an MoE model. When an MoE is
|
|
12248
|
-
// being processed then Accelerate/BLAS will not be involved, so capping would limit performance.
|
|
12249
|
-
if (n_tokens >= 32 && hparams.n_expert == 0 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas()) {
|
|
12250
|
-
n_threads = std::min(4, n_threads);
|
|
12251
|
-
}
|
|
12252
|
-
|
|
12253
12538
|
ggml_backend_sched_alloc_graph(lctx.sched, gf);
|
|
12254
12539
|
|
|
12255
12540
|
llama_set_inputs(lctx, u_batch);
|
|
@@ -12312,11 +12597,10 @@ static int llama_decode_internal(
|
|
|
12312
12597
|
ggml_backend_tensor_get_async(backend_embd, embd, embd_out, 0, n_outputs_new*n_embd*sizeof(float));
|
|
12313
12598
|
}
|
|
12314
12599
|
} break;
|
|
12315
|
-
case LLAMA_POOLING_TYPE_CLS:
|
|
12316
12600
|
case LLAMA_POOLING_TYPE_MEAN:
|
|
12601
|
+
case LLAMA_POOLING_TYPE_CLS:
|
|
12602
|
+
case LLAMA_POOLING_TYPE_LAST:
|
|
12317
12603
|
{
|
|
12318
|
-
GGML_ASSERT(strcmp(embd->name, "result_embd_pooled") == 0);
|
|
12319
|
-
|
|
12320
12604
|
// extract sequence embeddings
|
|
12321
12605
|
auto & embd_seq_out = lctx.embd_seq;
|
|
12322
12606
|
embd_seq_out.clear();
|
|
@@ -12930,107 +13214,142 @@ struct llm_bigram_bpe {
|
|
|
12930
13214
|
};
|
|
12931
13215
|
|
|
12932
13216
|
struct llm_tokenizer_bpe {
|
|
12933
|
-
llm_tokenizer_bpe(const llama_vocab & vocab): vocab(vocab) {
|
|
12934
|
-
|
|
12935
|
-
|
|
12936
|
-
|
|
12937
|
-
|
|
12938
|
-
|
|
12939
|
-
|
|
12940
|
-
|
|
12941
|
-
|
|
12942
|
-
|
|
12943
|
-
|
|
12944
|
-
|
|
12945
|
-
|
|
12946
|
-
|
|
12947
|
-
|
|
12948
|
-
|
|
12949
|
-
|
|
12950
|
-
|
|
12951
|
-
|
|
12952
|
-
|
|
12953
|
-
|
|
12954
|
-
|
|
12955
|
-
|
|
12956
|
-
|
|
12957
|
-
|
|
12958
|
-
|
|
12959
|
-
|
|
12960
|
-
|
|
12961
|
-
|
|
12962
|
-
|
|
12963
|
-
|
|
12964
|
-
|
|
12965
|
-
|
|
12966
|
-
|
|
12967
|
-
|
|
12968
|
-
|
|
12969
|
-
|
|
12970
|
-
|
|
12971
|
-
|
|
12972
|
-
|
|
12973
|
-
|
|
12974
|
-
|
|
12975
|
-
|
|
12976
|
-
|
|
12977
|
-
|
|
12978
|
-
|
|
12979
|
-
|
|
12980
|
-
|
|
12981
|
-
|
|
12982
|
-
|
|
12983
|
-
|
|
12984
|
-
|
|
12985
|
-
|
|
12986
|
-
|
|
12987
|
-
|
|
12988
|
-
|
|
12989
|
-
|
|
12990
|
-
|
|
12991
|
-
|
|
12992
|
-
|
|
12993
|
-
|
|
12994
|
-
|
|
12995
|
-
|
|
12996
|
-
|
|
12997
|
-
|
|
12998
|
-
|
|
12999
|
-
|
|
13000
|
-
|
|
13001
|
-
|
|
13002
|
-
|
|
13003
|
-
|
|
13004
|
-
|
|
13005
|
-
|
|
13006
|
-
|
|
13007
|
-
|
|
13008
|
-
|
|
13009
|
-
|
|
13010
|
-
|
|
13011
|
-
|
|
13012
|
-
|
|
13013
|
-
|
|
13014
|
-
|
|
13015
|
-
// "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
|
|
13016
|
-
"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
|
|
13017
|
-
});
|
|
13018
|
-
break;
|
|
13019
|
-
default:
|
|
13020
|
-
// default regex for BPE tokenization pre-processing
|
|
13021
|
-
word_collection = unicode_regex_split(text, {
|
|
13022
|
-
"[\\p{P}\\$\\+<=>\\^~\\|]+",
|
|
13023
|
-
"'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
|
|
13024
|
-
"\\p{N}+",
|
|
13025
|
-
"[0-9][0-9][0-9]",
|
|
13026
|
-
});
|
|
13027
|
-
break;
|
|
13028
|
-
}
|
|
13217
|
+
llm_tokenizer_bpe(const llama_vocab & vocab): vocab(vocab) {
|
|
13218
|
+
GGML_ASSERT(vocab.type == LLAMA_VOCAB_TYPE_BPE);
|
|
13219
|
+
switch (vocab.type_pre) {
|
|
13220
|
+
case LLAMA_VOCAB_PRE_TYPE_LLAMA3:
|
|
13221
|
+
regex_exprs = {
|
|
13222
|
+
// original regex from tokenizer.json
|
|
13223
|
+
//"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
|
|
13224
|
+
|
|
13225
|
+
// adapted: https://github.com/ggerganov/llama.cpp/pull/6920#issuecomment-2080233989
|
|
13226
|
+
"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
|
|
13227
|
+
};
|
|
13228
|
+
break;
|
|
13229
|
+
case LLAMA_VOCAB_PRE_TYPE_DBRX:
|
|
13230
|
+
case LLAMA_VOCAB_PRE_TYPE_SMAUG:
|
|
13231
|
+
regex_exprs = {
|
|
13232
|
+
// same as llama3
|
|
13233
|
+
"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
|
|
13234
|
+
};
|
|
13235
|
+
break;
|
|
13236
|
+
case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM:
|
|
13237
|
+
regex_exprs = {
|
|
13238
|
+
"[\r\n]",
|
|
13239
|
+
"\\s?[A-Za-zµÀ-ÖØ-öø-ƺƼ-ƿDŽ-ʓʕ-ʯͰ-ͳͶͷͻ-ͽͿΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԯԱ-ՖႠ-ჅᎠ-Ᏽᏸ-ᏽᲐ-ᲺᲽ-Ჿᴀ-ᴫᵫ-ᵷᵹ-ᶚḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℴℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-ⱻⱾ-ⳤⳫ-ⳮⳲⳳꙀ-ꙭꚀ-ꚛꜢ-ꝯꝱ-ꞇꞋ-ꞎꭰ-ꮿff-stﬓ-ﬗＡ-Ｚａ-ｚ𐐀-𐑏𐒰-𐓓𐓘-𐓻𐲀-𐲲𐳀-𐳲𑢠-𑣟𞤀-𞥃]+",
|
|
13240
|
+
"\\s?[!-/:-~！-／：-～‘-‟ -。]+",
|
|
13241
|
+
"\\s+$",
|
|
13242
|
+
"[一-龥ࠀ-一가-]+",
|
|
13243
|
+
"\\p{N}+",
|
|
13244
|
+
};
|
|
13245
|
+
break;
|
|
13246
|
+
case LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER:
|
|
13247
|
+
regex_exprs = {
|
|
13248
|
+
"[\r\n]",
|
|
13249
|
+
"\\s?\\p{L}+",
|
|
13250
|
+
"\\s?\\p{P}+",
|
|
13251
|
+
"[一-龥ࠀ-一가-]+",
|
|
13252
|
+
"\\p{N}",
|
|
13253
|
+
};
|
|
13254
|
+
break;
|
|
13255
|
+
case LLAMA_VOCAB_PRE_TYPE_FALCON:
|
|
13256
|
+
regex_exprs = {
|
|
13257
|
+
"[\\p{P}\\$\\+<=>\\^~\\|`]+",
|
|
13258
|
+
"'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
|
|
13259
|
+
"[0-9][0-9][0-9]",
|
|
13260
|
+
};
|
|
13261
|
+
break;
|
|
13262
|
+
case LLAMA_VOCAB_PRE_TYPE_MPT:
|
|
13263
|
+
// TODO: MPT pre-tokenization regexes are unknown
|
|
13264
|
+
// the following are close, but not exact. run the following:
|
|
13265
|
+
// ./bin/test-tokenizer-0 ../models/ggml-vocab-mpt.gguf
|
|
13266
|
+
GGML_ASSERT("MPT pre-tokenization regexes are unknown - fixes needed");
|
|
13267
|
+
regex_exprs = {
|
|
13268
|
+
"\\s?\\p{L}+",
|
|
13269
|
+
"\\s?\\p{P}+",
|
|
13270
|
+
"'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
|
|
13271
|
+
};
|
|
13272
|
+
break;
|
|
13273
|
+
case LLAMA_VOCAB_PRE_TYPE_STARCODER:
|
|
13274
|
+
case LLAMA_VOCAB_PRE_TYPE_REFACT:
|
|
13275
|
+
case LLAMA_VOCAB_PRE_TYPE_COMMAND_R:
|
|
13276
|
+
regex_exprs = {
|
|
13277
|
+
"\\p{N}",
|
|
13278
|
+
"'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
|
|
13279
|
+
};
|
|
13280
|
+
break;
|
|
13281
|
+
case LLAMA_VOCAB_PRE_TYPE_GPT2:
|
|
13282
|
+
case LLAMA_VOCAB_PRE_TYPE_OLMO:
|
|
13283
|
+
regex_exprs = {
|
|
13284
|
+
"'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
|
|
13285
|
+
};
|
|
13286
|
+
break;
|
|
13287
|
+
case LLAMA_VOCAB_PRE_TYPE_STABLELM2:
|
|
13288
|
+
case LLAMA_VOCAB_PRE_TYPE_QWEN2:
|
|
13289
|
+
regex_exprs = {
|
|
13290
|
+
// original regex from tokenizer.json
|
|
13291
|
+
// "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
|
|
13292
|
+
"(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
|
|
13293
|
+
};
|
|
13294
|
+
break;
|
|
13295
|
+
case LLAMA_VOCAB_PRE_TYPE_PORO:
|
|
13296
|
+
regex_exprs = {
|
|
13297
|
+
" ?[^(\\s|.,!?…。，、।۔،)]+",
|
|
13298
|
+
};
|
|
13029
13299
|
break;
|
|
13030
13300
|
default:
|
|
13031
|
-
|
|
13301
|
+
// default regex for BPE tokenization pre-processing
|
|
13302
|
+
regex_exprs = {
|
|
13303
|
+
"[\\p{P}\\$\\+<=>\\^~\\|]+",
|
|
13304
|
+
"'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
|
|
13305
|
+
"\\p{N}+",
|
|
13306
|
+
"[0-9][0-9][0-9]",
|
|
13307
|
+
};
|
|
13032
13308
|
break;
|
|
13033
13309
|
}
|
|
13310
|
+
}
|
|
13311
|
+
|
|
13312
|
+
void append(const llama_vocab::id token_id, std::vector<llama_vocab::id> & output) const {
|
|
13313
|
+
output.push_back(token_id);
|
|
13314
|
+
}
|
|
13315
|
+
|
|
13316
|
+
bool append_bos(std::vector<llama_vocab::id> & output) const {
|
|
13317
|
+
if (vocab.tokenizer_add_bos) {
|
|
13318
|
+
GGML_ASSERT(vocab.special_bos_id != -1);
|
|
13319
|
+
output.push_back(vocab.special_bos_id);
|
|
13320
|
+
return true;
|
|
13321
|
+
}
|
|
13322
|
+
return false;
|
|
13323
|
+
}
|
|
13324
|
+
|
|
13325
|
+
bool append_eos(std::vector<llama_vocab::id> & output) const {
|
|
13326
|
+
if (vocab.tokenizer_add_eos) {
|
|
13327
|
+
GGML_ASSERT(vocab.special_eos_id != -1);
|
|
13328
|
+
output.push_back(vocab.special_eos_id);
|
|
13329
|
+
return true;
|
|
13330
|
+
}
|
|
13331
|
+
return false;
|
|
13332
|
+
}
|
|
13333
|
+
|
|
13334
|
+
void check_double_bos_eos(const std::vector<llama_vocab::id> & output) const {
|
|
13335
|
+
if (vocab.tokenizer_add_bos && output.size() >= 2 && output[1] == vocab.special_bos_id) {
|
|
13336
|
+
LLAMA_LOG_WARN(
|
|
13337
|
+
"%s: Added a BOS token to the prompt as specified by the model but the prompt "
|
|
13338
|
+
"also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
|
|
13339
|
+
"Are you sure this is what you want?\n", __FUNCTION__);
|
|
13340
|
+
}
|
|
13341
|
+
if (vocab.tokenizer_add_eos && output.size() >= 2 && *(output.end()-2) == vocab.special_eos_id) {
|
|
13342
|
+
LLAMA_LOG_WARN(
|
|
13343
|
+
"%s: Added a EOS token to the prompt as specified by the model but the prompt "
|
|
13344
|
+
"also ends with a EOS token. So now the final prompt ends with 2 EOS tokens. "
|
|
13345
|
+
"Are you sure this is what you want?\n", __FUNCTION__);
|
|
13346
|
+
}
|
|
13347
|
+
}
|
|
13348
|
+
|
|
13349
|
+
void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
|
|
13350
|
+
int final_prev_index = -1;
|
|
13351
|
+
|
|
13352
|
+
const auto word_collection = unicode_regex_split(text, regex_exprs);
|
|
13034
13353
|
|
|
13035
13354
|
symbols_final.clear();
|
|
13036
13355
|
|
|
@@ -13041,7 +13360,7 @@ struct llm_tokenizer_bpe {
|
|
|
13041
13360
|
int index = 0;
|
|
13042
13361
|
size_t offset = 0;
|
|
13043
13362
|
|
|
13044
|
-
if (
|
|
13363
|
+
if (vocab.tokenizer_ignore_merges && vocab.token_to_id.find(word) != vocab.token_to_id.end()) {
|
|
13045
13364
|
symbols.emplace_back(llm_symbol{-1, -1, word.c_str(), word.size()});
|
|
13046
13365
|
offset = word.size();
|
|
13047
13366
|
}
|
|
@@ -13122,10 +13441,9 @@ struct llm_tokenizer_bpe {
|
|
|
13122
13441
|
for (auto j = str.begin(); j != str.end(); ++j) {
|
|
13123
13442
|
std::string byte_str(1, *j);
|
|
13124
13443
|
auto token_multibyte = vocab.token_to_id.find(byte_str);
|
|
13125
|
-
if (token_multibyte
|
|
13126
|
-
|
|
13444
|
+
if (token_multibyte != vocab.token_to_id.end()) {
|
|
13445
|
+
output.push_back(token_multibyte->second);
|
|
13127
13446
|
}
|
|
13128
|
-
output.push_back((*token_multibyte).second);
|
|
13129
13447
|
}
|
|
13130
13448
|
} else {
|
|
13131
13449
|
output.push_back((*token).second);
|
|
@@ -13164,6 +13482,8 @@ private:
|
|
|
13164
13482
|
|
|
13165
13483
|
const llama_vocab & vocab;
|
|
13166
13484
|
|
|
13485
|
+
std::vector<std::string> regex_exprs;
|
|
13486
|
+
|
|
13167
13487
|
std::vector<llm_symbol> symbols;
|
|
13168
13488
|
std::vector<llm_symbol> symbols_final;
|
|
13169
13489
|
|
|
@@ -13173,7 +13493,7 @@ private:
|
|
|
13173
13493
|
struct llm_tokenizer_wpm {
|
|
13174
13494
|
llm_tokenizer_wpm(const llama_vocab & vocab): vocab(vocab) {}
|
|
13175
13495
|
|
|
13176
|
-
void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
|
|
13496
|
+
void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) const {
|
|
13177
13497
|
const auto & token_map = vocab.token_to_id;
|
|
13178
13498
|
|
|
13179
13499
|
// normalize and split by whitespace
|
|
@@ -13182,7 +13502,7 @@ struct llm_tokenizer_wpm {
|
|
|
13182
13502
|
// bos token prepended already
|
|
13183
13503
|
|
|
13184
13504
|
// find the longest tokens that form the words
|
|
13185
|
-
for (const std::string &word : words) {
|
|
13505
|
+
for (const std::string & word : words) {
|
|
13186
13506
|
// skip empty words
|
|
13187
13507
|
if (word.size() == 0) {
|
|
13188
13508
|
continue;
|
|
@@ -13199,7 +13519,7 @@ struct llm_tokenizer_wpm {
|
|
|
13199
13519
|
for (int i = 0; i < n; ++i) {
|
|
13200
13520
|
// loop through possible match length
|
|
13201
13521
|
bool match = false;
|
|
13202
|
-
for (int j = n; j > i; j--) {
|
|
13522
|
+
for (int j = std::min(n, i + vocab.max_token_len + 1); j > i; j--) {
|
|
13203
13523
|
auto it = token_map.find(word1.substr(i, j - i));
|
|
13204
13524
|
if (it != token_map.end()) {
|
|
13205
13525
|
output.push_back(it->second);
|
|
@@ -13222,11 +13542,12 @@ struct llm_tokenizer_wpm {
|
|
|
13222
13542
|
}
|
|
13223
13543
|
}
|
|
13224
13544
|
|
|
13225
|
-
|
|
13545
|
+
// TODO: reduce string copies by using cpts_offs array
|
|
13546
|
+
std::vector<std::string> preprocess(const std::string & text) const {
|
|
13226
13547
|
const std::vector<uint32_t> cpts_nfd = unicode_cpts_normalize_nfd(unicode_cpts_from_utf8(text));
|
|
13227
13548
|
std::vector<std::string> words(1, "");
|
|
13228
13549
|
|
|
13229
|
-
for (const
|
|
13550
|
+
for (const uint32_t cpt : cpts_nfd) {
|
|
13230
13551
|
const auto flags = unicode_cpt_flags(cpt);
|
|
13231
13552
|
|
|
13232
13553
|
if (flags.is_whitespace) {
|
|
@@ -13444,7 +13765,7 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
|
|
|
13444
13765
|
|
|
13445
13766
|
bool is_prev_special = false;
|
|
13446
13767
|
|
|
13447
|
-
if (add_special && vocab.
|
|
13768
|
+
if (add_special && vocab.tokenizer_add_bos) {
|
|
13448
13769
|
GGML_ASSERT(vocab.special_bos_id != -1);
|
|
13449
13770
|
output.push_back(vocab.special_bos_id);
|
|
13450
13771
|
is_prev_special = true;
|
|
@@ -13454,7 +13775,7 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
|
|
|
13454
13775
|
if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
|
|
13455
13776
|
auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
|
|
13456
13777
|
|
|
13457
|
-
if (vocab.
|
|
13778
|
+
if (vocab.tokenizer_add_space_prefix) {
|
|
13458
13779
|
if (!output.size() || is_prev_special) { // prefix with space if first token
|
|
13459
13780
|
raw_text = " " + raw_text;
|
|
13460
13781
|
}
|
|
@@ -13472,23 +13793,24 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
|
|
|
13472
13793
|
}
|
|
13473
13794
|
}
|
|
13474
13795
|
|
|
13475
|
-
if (add_special && vocab.
|
|
13796
|
+
if (add_special && vocab.tokenizer_add_bos && output.size() >= 2 && output[1] == vocab.special_bos_id) {
|
|
13476
13797
|
LLAMA_LOG_WARN(
|
|
13477
13798
|
"%s: Added a BOS token to the prompt as specified by the model but the prompt "
|
|
13478
13799
|
"also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
|
|
13479
13800
|
"Are you sure this is what you want?\n", __FUNCTION__);
|
|
13480
13801
|
}
|
|
13481
13802
|
|
|
13482
|
-
if (add_special && vocab.
|
|
13803
|
+
if (add_special && vocab.tokenizer_add_eos) {
|
|
13483
13804
|
GGML_ASSERT(vocab.special_eos_id != -1);
|
|
13484
13805
|
output.push_back(vocab.special_eos_id);
|
|
13485
13806
|
}
|
|
13486
13807
|
} break;
|
|
13487
13808
|
case LLAMA_VOCAB_TYPE_BPE:
|
|
13488
13809
|
{
|
|
13489
|
-
|
|
13490
|
-
|
|
13491
|
-
|
|
13810
|
+
llm_tokenizer_bpe tokenizer(vocab);
|
|
13811
|
+
|
|
13812
|
+
if (add_special) {
|
|
13813
|
+
tokenizer.append_bos(output);
|
|
13492
13814
|
}
|
|
13493
13815
|
|
|
13494
13816
|
for (const auto & fragment : fragment_buffer) {
|
|
@@ -13498,23 +13820,15 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
|
|
|
13498
13820
|
#ifdef PRETOKENIZERDEBUG
|
|
13499
13821
|
LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());
|
|
13500
13822
|
#endif
|
|
13501
|
-
llm_tokenizer_bpe tokenizer(vocab);
|
|
13502
13823
|
tokenizer.tokenize(raw_text, output);
|
|
13503
13824
|
} else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
|
|
13504
|
-
|
|
13825
|
+
tokenizer.append(fragment.token, output);
|
|
13505
13826
|
}
|
|
13506
13827
|
}
|
|
13507
13828
|
|
|
13508
|
-
if (add_special
|
|
13509
|
-
|
|
13510
|
-
|
|
13511
|
-
"also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. "
|
|
13512
|
-
"Are you sure this is what you want?\n", __FUNCTION__);
|
|
13513
|
-
}
|
|
13514
|
-
|
|
13515
|
-
if (add_special && vocab.special_add_eos == 1) {
|
|
13516
|
-
GGML_ASSERT(vocab.special_add_eos != -1);
|
|
13517
|
-
output.push_back(vocab.special_eos_id);
|
|
13829
|
+
if (add_special) {
|
|
13830
|
+
tokenizer.append_eos(output);
|
|
13831
|
+
tokenizer.check_double_bos_eos(output);
|
|
13518
13832
|
}
|
|
13519
13833
|
} break;
|
|
13520
13834
|
case LLAMA_VOCAB_TYPE_WPM:
|
|
@@ -13524,6 +13838,8 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
|
|
|
13524
13838
|
output.push_back(vocab.special_cls_id);
|
|
13525
13839
|
}
|
|
13526
13840
|
|
|
13841
|
+
llm_tokenizer_wpm tokenizer(vocab);
|
|
13842
|
+
|
|
13527
13843
|
for (const auto & fragment : fragment_buffer) {
|
|
13528
13844
|
if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) {
|
|
13529
13845
|
auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length);
|
|
@@ -13531,7 +13847,6 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
|
|
|
13531
13847
|
#ifdef PRETOKENIZERDEBUG
|
|
13532
13848
|
LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());
|
|
13533
13849
|
#endif
|
|
13534
|
-
llm_tokenizer_wpm tokenizer(vocab);
|
|
13535
13850
|
tokenizer.tokenize(raw_text, output);
|
|
13536
13851
|
} else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN)
|
|
13537
13852
|
output.push_back(fragment.token);
|
|
@@ -13631,7 +13946,7 @@ static std::pair<bool, const llama_grammar_element *> llama_grammar_match_char(
|
|
|
13631
13946
|
const uint32_t chr) {
|
|
13632
13947
|
|
|
13633
13948
|
bool found = false;
|
|
13634
|
-
bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR;
|
|
13949
|
+
bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR || pos->type == LLAMA_GRETYPE_CHAR_ANY;
|
|
13635
13950
|
|
|
13636
13951
|
GGML_ASSERT(is_positive_char || pos->type == LLAMA_GRETYPE_CHAR_NOT); // NOLINT
|
|
13637
13952
|
|
|
@@ -13640,6 +13955,10 @@ static std::pair<bool, const llama_grammar_element *> llama_grammar_match_char(
|
|
|
13640
13955
|
// inclusive range, e.g. [a-z]
|
|
13641
13956
|
found = found || (pos->value <= chr && chr <= pos[1].value);
|
|
13642
13957
|
pos += 2;
|
|
13958
|
+
} else if (pos->type == LLAMA_GRETYPE_CHAR_ANY) {
|
|
13959
|
+
// Any character matches "."
|
|
13960
|
+
found = true;
|
|
13961
|
+
pos += 1;
|
|
13643
13962
|
} else {
|
|
13644
13963
|
// exact char match, e.g. [a] or "a"
|
|
13645
13964
|
found = found || pos->value == chr;
|
|
@@ -13657,7 +13976,7 @@ static bool llama_grammar_match_partial_char(
|
|
|
13657
13976
|
const llama_grammar_element * pos,
|
|
13658
13977
|
const llama_partial_utf8 partial_utf8) {
|
|
13659
13978
|
|
|
13660
|
-
bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR;
|
|
13979
|
+
bool is_positive_char = pos->type == LLAMA_GRETYPE_CHAR || pos->type == LLAMA_GRETYPE_CHAR_ANY;
|
|
13661
13980
|
GGML_ASSERT(is_positive_char || pos->type == LLAMA_GRETYPE_CHAR_NOT);
|
|
13662
13981
|
|
|
13663
13982
|
uint32_t partial_value = partial_utf8.value;
|
|
@@ -13687,6 +14006,9 @@ static bool llama_grammar_match_partial_char(
|
|
|
13687
14006
|
return is_positive_char;
|
|
13688
14007
|
}
|
|
13689
14008
|
pos += 2;
|
|
14009
|
+
} else if (pos->type == LLAMA_GRETYPE_CHAR_ANY) {
|
|
14010
|
+
// Any character matches "."
|
|
14011
|
+
return true;
|
|
13690
14012
|
} else {
|
|
13691
14013
|
// exact char match, e.g. [a] or "a"
|
|
13692
14014
|
if (low <= pos->value && pos->value <= high) {
|
|
@@ -13747,6 +14069,7 @@ static void llama_grammar_advance_stack(
|
|
|
13747
14069
|
}
|
|
13748
14070
|
case LLAMA_GRETYPE_CHAR:
|
|
13749
14071
|
case LLAMA_GRETYPE_CHAR_NOT:
|
|
14072
|
+
case LLAMA_GRETYPE_CHAR_ANY:
|
|
13750
14073
|
if (std::find(new_stacks.begin(), new_stacks.end(), stack) == new_stacks.end()) {
|
|
13751
14074
|
// only add the stack if it's not a duplicate of one we already have
|
|
13752
14075
|
new_stacks.emplace_back(stack);
|
|
@@ -15220,6 +15543,14 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
|
|
|
15220
15543
|
if (imatrix_data) {
|
|
15221
15544
|
LLAMA_LOG_INFO("================================ Have weights data with %d entries\n",int(imatrix_data->size()));
|
|
15222
15545
|
qs.has_imatrix = true;
|
|
15546
|
+
// check imatrix for nans or infs
|
|
15547
|
+
for (const auto & kv : *imatrix_data) {
|
|
15548
|
+
for (float f : kv.second) {
|
|
15549
|
+
if (!std::isfinite(f)) {
|
|
15550
|
+
throw std::runtime_error(format("imatrix contains non-finite value %f\n", f));
|
|
15551
|
+
}
|
|
15552
|
+
}
|
|
15553
|
+
}
|
|
15223
15554
|
}
|
|
15224
15555
|
}
|
|
15225
15556
|
|
|
@@ -16024,6 +16355,11 @@ struct llama_context * llama_new_context_with_model(
|
|
|
16024
16355
|
params.flash_attn = false;
|
|
16025
16356
|
}
|
|
16026
16357
|
|
|
16358
|
+
if (params.flash_attn && model->hparams.n_embd_head_k != model->hparams.n_embd_head_v) {
|
|
16359
|
+
LLAMA_LOG_WARN("%s: flash_attn requires n_embd_head_k == n_embd_head_v - forcing off\n", __func__);
|
|
16360
|
+
params.flash_attn = false;
|
|
16361
|
+
}
|
|
16362
|
+
|
|
16027
16363
|
if (params.type_v != GGML_TYPE_F16 && !params.flash_attn) {
|
|
16028
16364
|
LLAMA_LOG_ERROR("%s: V cache quantization requires flash_attn\n", __func__);
|
|
16029
16365
|
return nullptr;
|
|
@@ -16195,8 +16531,7 @@ struct llama_context * llama_new_context_with_model(
|
|
|
16195
16531
|
if (model->split_mode == LLAMA_SPLIT_MODE_NONE || model->split_mode == LLAMA_SPLIT_MODE_ROW) {
|
|
16196
16532
|
ggml_backend_t backend = ggml_backend_sycl_init(model->main_gpu);
|
|
16197
16533
|
if (backend == nullptr) {
|
|
16198
|
-
|
|
16199
|
-
LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d (index %d) backend\n", __func__, main_gpu_id, model->main_gpu);
|
|
16534
|
+
LLAMA_LOG_ERROR("%s: failed to initialize SYCL%d backend\n", __func__, model->main_gpu);
|
|
16200
16535
|
llama_free(ctx);
|
|
16201
16536
|
return nullptr;
|
|
16202
16537
|
}
|
|
@@ -16226,6 +16561,16 @@ struct llama_context * llama_new_context_with_model(
|
|
|
16226
16561
|
ctx->backends.push_back(backend);
|
|
16227
16562
|
}
|
|
16228
16563
|
#endif
|
|
16564
|
+
|
|
16565
|
+
#ifdef GGML_USE_BLAS
|
|
16566
|
+
ctx->backend_blas = ggml_backend_blas_init();
|
|
16567
|
+
if (ctx->backend_blas == nullptr) {
|
|
16568
|
+
LLAMA_LOG_WARN("%s: failed to initialize BLAS backend\n", __func__);
|
|
16569
|
+
} else {
|
|
16570
|
+
ctx->backends.push_back(ctx->backend_blas);
|
|
16571
|
+
}
|
|
16572
|
+
#endif
|
|
16573
|
+
|
|
16229
16574
|
#if defined(GGML_USE_RPC)
|
|
16230
16575
|
if (model->n_gpu_layers > 0) {
|
|
16231
16576
|
for (const auto & endpoint : model->rpc_servers) {
|
|
@@ -17814,6 +18159,10 @@ void llama_set_abort_callback(struct llama_context * ctx, bool (*abort_callback)
|
|
|
17814
18159
|
ctx->abort_callback_data = abort_callback_data;
|
|
17815
18160
|
}
|
|
17816
18161
|
|
|
18162
|
+
void llama_set_embeddings(struct llama_context * ctx, bool embeddings) {
|
|
18163
|
+
ctx->cparams.embeddings = embeddings;
|
|
18164
|
+
}
|
|
18165
|
+
|
|
17817
18166
|
void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn) {
|
|
17818
18167
|
ctx->cparams.causal_attn = causal_attn;
|
|
17819
18168
|
}
|
|
@@ -18057,11 +18406,11 @@ llama_token llama_token_nl(const struct llama_model * model) {
|
|
|
18057
18406
|
}
|
|
18058
18407
|
|
|
18059
18408
|
int32_t llama_add_bos_token(const struct llama_model * model) {
|
|
18060
|
-
return model->vocab.
|
|
18409
|
+
return model->vocab.tokenizer_add_bos;
|
|
18061
18410
|
}
|
|
18062
18411
|
|
|
18063
18412
|
int32_t llama_add_eos_token(const struct llama_model * model) {
|
|
18064
|
-
return model->vocab.
|
|
18413
|
+
return model->vocab.tokenizer_add_eos;
|
|
18065
18414
|
}
|
|
18066
18415
|
|
|
18067
18416
|
llama_token llama_token_prefix(const struct llama_model * model) {
|