npm - @fugood/llama.node - Versions diffs - 0.3.6 → 0.3.8 - Mend

@fugood/llama.node 0.3.6 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (186) hide show

package/README.md +17 -2
package/bin/darwin/arm64/llama-node.node +0 -0
package/bin/darwin/x64/llama-node.node +0 -0
package/bin/linux/arm64/llama-node.node +0 -0
package/bin/linux/x64/llama-node.node +0 -0
package/bin/linux-cuda/arm64/llama-node.node +0 -0
package/bin/linux-cuda/x64/llama-node.node +0 -0
package/bin/linux-vulkan/arm64/llama-node.node +0 -0
package/bin/linux-vulkan/x64/llama-node.node +0 -0
package/bin/win32/arm64/llama-node.node +0 -0
package/bin/win32/arm64/node.lib +0 -0
package/bin/win32/x64/llama-node.node +0 -0
package/bin/win32/x64/node.lib +0 -0
package/bin/win32-vulkan/arm64/llama-node.node +0 -0
package/bin/win32-vulkan/arm64/node.lib +0 -0
package/bin/win32-vulkan/x64/llama-node.node +0 -0
package/bin/win32-vulkan/x64/node.lib +0 -0
package/lib/binding.ts +3 -1
package/lib/index.js +16 -1
package/lib/index.ts +16 -0
package/package.json +1 -1
package/src/EmbeddingWorker.cpp +4 -3
package/src/LlamaCompletionWorker.cpp +4 -2
package/src/LlamaContext.cpp +61 -6
package/src/LlamaContext.h +1 -0
package/src/common.hpp +6 -11
package/src/llama.cpp/.github/workflows/build.yml +19 -17
package/src/llama.cpp/.github/workflows/docker.yml +77 -30
package/src/llama.cpp/.github/workflows/editorconfig.yml +3 -1
package/src/llama.cpp/.github/workflows/server.yml +22 -3
package/src/llama.cpp/CMakeLists.txt +49 -24
package/src/llama.cpp/common/arg.cpp +82 -26
package/src/llama.cpp/common/arg.h +3 -0
package/src/llama.cpp/common/common.cpp +192 -72
package/src/llama.cpp/common/common.h +51 -18
package/src/llama.cpp/common/ngram-cache.cpp +12 -12
package/src/llama.cpp/common/ngram-cache.h +2 -2
package/src/llama.cpp/common/sampling.cpp +11 -6
package/src/llama.cpp/common/speculative.cpp +18 -15
package/src/llama.cpp/docs/build.md +2 -0
package/src/llama.cpp/examples/batched/batched.cpp +9 -7
package/src/llama.cpp/examples/batched-bench/batched-bench.cpp +3 -3
package/src/llama.cpp/examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp +10 -8
package/src/llama.cpp/examples/cvector-generator/cvector-generator.cpp +11 -8
package/src/llama.cpp/examples/cvector-generator/mean.hpp +1 -1
package/src/llama.cpp/examples/cvector-generator/pca.hpp +1 -1
package/src/llama.cpp/examples/embedding/embedding.cpp +8 -7
package/src/llama.cpp/examples/eval-callback/eval-callback.cpp +7 -6
package/src/llama.cpp/examples/export-lora/export-lora.cpp +8 -7
package/src/llama.cpp/examples/gguf/gguf.cpp +10 -6
package/src/llama.cpp/examples/gguf-hash/gguf-hash.cpp +1 -0
package/src/llama.cpp/examples/gguf-split/gguf-split.cpp +8 -7
package/src/llama.cpp/examples/gritlm/gritlm.cpp +13 -10
package/src/llama.cpp/examples/imatrix/imatrix.cpp +13 -12
package/src/llama.cpp/examples/infill/infill.cpp +23 -24
package/src/llama.cpp/examples/llama-bench/llama-bench.cpp +44 -13
package/src/llama.cpp/examples/llama.android/llama/src/main/cpp/llama-android.cpp +11 -6
package/src/llama.cpp/examples/llava/clip.cpp +4 -2
package/src/llama.cpp/examples/llava/llava-cli.cpp +9 -6
package/src/llama.cpp/examples/llava/llava.cpp +2 -2
package/src/llama.cpp/examples/llava/minicpmv-cli.cpp +8 -4
package/src/llama.cpp/examples/llava/qwen2vl-cli.cpp +11 -8
package/src/llama.cpp/examples/lookahead/lookahead.cpp +6 -7
package/src/llama.cpp/examples/lookup/lookup-create.cpp +4 -9
package/src/llama.cpp/examples/lookup/lookup-stats.cpp +3 -7
package/src/llama.cpp/examples/lookup/lookup.cpp +5 -6
package/src/llama.cpp/examples/main/main.cpp +51 -29
package/src/llama.cpp/examples/parallel/parallel.cpp +5 -6
package/src/llama.cpp/examples/passkey/passkey.cpp +7 -5
package/src/llama.cpp/examples/perplexity/perplexity.cpp +37 -23
package/src/llama.cpp/examples/quantize-stats/quantize-stats.cpp +12 -14
package/src/llama.cpp/examples/retrieval/retrieval.cpp +8 -8
package/src/llama.cpp/examples/rpc/rpc-server.cpp +12 -0
package/src/llama.cpp/examples/run/CMakeLists.txt +1 -1
package/src/llama.cpp/examples/run/linenoise.cpp/linenoise.cpp +1351 -0
package/src/llama.cpp/examples/run/linenoise.cpp/linenoise.h +114 -0
package/src/llama.cpp/examples/run/run.cpp +175 -61
package/src/llama.cpp/examples/save-load-state/save-load-state.cpp +4 -25
package/src/llama.cpp/examples/server/CMakeLists.txt +1 -0
package/src/llama.cpp/examples/server/httplib.h +1295 -409
package/src/llama.cpp/examples/server/server.cpp +387 -181
package/src/llama.cpp/examples/server/tests/requirements.txt +1 -0
package/src/llama.cpp/examples/server/utils.hpp +170 -58
package/src/llama.cpp/examples/simple/simple.cpp +9 -8
package/src/llama.cpp/examples/simple-chat/simple-chat.cpp +16 -12
package/src/llama.cpp/examples/speculative/speculative.cpp +22 -23
package/src/llama.cpp/examples/speculative-simple/speculative-simple.cpp +8 -12
package/src/llama.cpp/examples/tokenize/tokenize.cpp +17 -5
package/src/llama.cpp/examples/tts/tts.cpp +64 -23
package/src/llama.cpp/ggml/CMakeLists.txt +5 -21
package/src/llama.cpp/ggml/include/ggml-backend.h +2 -0
package/src/llama.cpp/ggml/include/ggml-cpp.h +1 -0
package/src/llama.cpp/ggml/include/ggml.h +36 -145
package/src/llama.cpp/ggml/include/gguf.h +202 -0
package/src/llama.cpp/ggml/src/CMakeLists.txt +6 -3
package/src/llama.cpp/ggml/src/ggml-alloc.c +5 -0
package/src/llama.cpp/ggml/src/ggml-backend-impl.h +0 -1
package/src/llama.cpp/ggml/src/ggml-backend-reg.cpp +79 -49
package/src/llama.cpp/ggml/src/ggml-backend.cpp +5 -2
package/src/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt +33 -23
package/src/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +57 -72
package/src/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-quants.c +87 -2
package/src/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +335 -66
package/src/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp +10 -2
package/src/llama.cpp/ggml/src/ggml-cpu/llamafile/sgemm.cpp +1090 -378
package/src/llama.cpp/ggml/src/ggml-cpu/llamafile/sgemm.h +2 -2
package/src/llama.cpp/ggml/src/ggml-cuda/vendors/cuda.h +1 -0
package/src/llama.cpp/ggml/src/ggml-cuda/vendors/hip.h +3 -0
package/src/llama.cpp/ggml/src/ggml-cuda/vendors/musa.h +3 -0
package/src/llama.cpp/ggml/src/ggml-hip/CMakeLists.txt +3 -1
package/src/llama.cpp/ggml/src/ggml-impl.h +11 -16
package/src/llama.cpp/ggml/src/ggml-metal/CMakeLists.txt +16 -0
package/src/llama.cpp/ggml/src/ggml-opencl/ggml-opencl.cpp +6 -6
package/src/llama.cpp/ggml/src/ggml-rpc/ggml-rpc.cpp +154 -35
package/src/llama.cpp/ggml/src/ggml-sycl/backend.hpp +1 -0
package/src/llama.cpp/ggml/src/ggml-sycl/common.cpp +9 -3
package/src/llama.cpp/ggml/src/ggml-sycl/common.hpp +18 -0
package/src/llama.cpp/ggml/src/ggml-sycl/concat.cpp +3 -2
package/src/llama.cpp/ggml/src/ggml-sycl/concat.hpp +1 -2
package/src/llama.cpp/ggml/src/ggml-sycl/conv.cpp +3 -2
package/src/llama.cpp/ggml/src/ggml-sycl/conv.hpp +1 -2
package/src/llama.cpp/ggml/src/ggml-sycl/dpct/helper.hpp +40 -95
package/src/llama.cpp/ggml/src/ggml-sycl/element_wise.cpp +48 -48
package/src/llama.cpp/ggml/src/ggml-sycl/element_wise.hpp +24 -24
package/src/llama.cpp/ggml/src/ggml-sycl/ggml-sycl.cpp +238 -164
package/src/llama.cpp/ggml/src/ggml-sycl/gla.cpp +105 -0
package/src/llama.cpp/ggml/src/ggml-sycl/gla.hpp +8 -0
package/src/llama.cpp/ggml/src/ggml-sycl/outprod.cpp +3 -3
package/src/llama.cpp/ggml/src/ggml-sycl/outprod.hpp +1 -2
package/src/llama.cpp/ggml/src/ggml-sycl/tsembd.cpp +3 -2
package/src/llama.cpp/ggml/src/ggml-sycl/tsembd.hpp +1 -2
package/src/llama.cpp/ggml/src/ggml-sycl/wkv6.cpp +7 -5
package/src/llama.cpp/ggml/src/ggml-sycl/wkv6.hpp +1 -2
package/src/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt +74 -4
package/src/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp +314 -116
package/src/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +4 -2
package/src/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +9 -3
package/src/llama.cpp/ggml/src/ggml.c +117 -1327
package/src/llama.cpp/ggml/src/gguf.cpp +1329 -0
package/src/llama.cpp/include/llama-cpp.h +6 -1
package/src/llama.cpp/include/llama.h +138 -75
package/src/llama.cpp/src/CMakeLists.txt +13 -1
package/src/llama.cpp/src/llama-adapter.cpp +347 -0
package/src/llama.cpp/src/llama-adapter.h +74 -0
package/src/llama.cpp/src/llama-arch.cpp +1487 -0
package/src/llama.cpp/src/llama-arch.h +400 -0
package/src/llama.cpp/src/llama-batch.cpp +368 -0
package/src/llama.cpp/src/llama-batch.h +88 -0
package/src/llama.cpp/src/llama-chat.cpp +578 -0
package/src/llama.cpp/src/llama-chat.h +52 -0
package/src/llama.cpp/src/llama-context.cpp +1775 -0
package/src/llama.cpp/src/llama-context.h +128 -0
package/src/llama.cpp/src/llama-cparams.cpp +1 -0
package/src/llama.cpp/src/llama-cparams.h +37 -0
package/src/llama.cpp/src/llama-grammar.cpp +5 -4
package/src/llama.cpp/src/llama-grammar.h +3 -1
package/src/llama.cpp/src/llama-hparams.cpp +71 -0
package/src/llama.cpp/src/llama-hparams.h +139 -0
package/src/llama.cpp/src/llama-impl.cpp +167 -0
package/src/llama.cpp/src/llama-impl.h +16 -136
package/src/llama.cpp/src/llama-kv-cache.cpp +718 -0
package/src/llama.cpp/src/llama-kv-cache.h +218 -0
package/src/llama.cpp/src/llama-mmap.cpp +589 -0
package/src/llama.cpp/src/llama-mmap.h +67 -0
package/src/llama.cpp/src/llama-model-loader.cpp +1124 -0
package/src/llama.cpp/src/llama-model-loader.h +167 -0
package/src/llama.cpp/src/llama-model.cpp +3953 -0
package/src/llama.cpp/src/llama-model.h +370 -0
package/src/llama.cpp/src/llama-quant.cpp +934 -0
package/src/llama.cpp/src/llama-quant.h +1 -0
package/src/llama.cpp/src/llama-sampling.cpp +147 -32
package/src/llama.cpp/src/llama-sampling.h +3 -19
package/src/llama.cpp/src/llama-vocab.cpp +1832 -575
package/src/llama.cpp/src/llama-vocab.h +97 -142
package/src/llama.cpp/src/llama.cpp +7160 -20314
package/src/llama.cpp/src/unicode.cpp +8 -3
package/src/llama.cpp/tests/CMakeLists.txt +2 -0
package/src/llama.cpp/tests/test-autorelease.cpp +3 -3
package/src/llama.cpp/tests/test-backend-ops.cpp +370 -59
package/src/llama.cpp/tests/test-chat-template.cpp +162 -125
package/src/llama.cpp/tests/test-gguf.cpp +222 -187
package/src/llama.cpp/tests/test-model-load-cancel.cpp +1 -1
package/src/llama.cpp/tests/test-sampling.cpp +0 -1
package/src/llama.cpp/tests/test-tokenizer-0.cpp +4 -4
package/src/llama.cpp/tests/test-tokenizer-1-bpe.cpp +9 -7
package/src/llama.cpp/tests/test-tokenizer-1-spm.cpp +8 -6

package/src/llama.cpp/src/llama-vocab.h CHANGED Viewed

@@ -1,170 +1,125 @@
 #pragma once
-#include "llama-impl.h"
+#include "llama.h"
 #include <string>
 #include <vector>
-#include <unordered_map>
-#include <map>
-#include <set>
+#include <memory>
-struct llm_tokenizer;
+struct LLM_KV;
+struct llama_model_loader;
 struct llama_vocab {
-    using id    = llama_token;
-    using token = std::string;
-    using tattr = llama_token_attr;
     struct token_data {
-        token text;
-        float score;
-        tattr attr;
+        std::string      text;
+        float            score;
+        llama_token_attr attr;
     };
-    uint32_t n_vocab = 0; // TODO: not great because has to keep in sync with hparams.n_vocab
+    llama_vocab();
+    ~llama_vocab();
-    enum llama_vocab_type     type     = LLAMA_VOCAB_TYPE_SPM;
-    enum llama_vocab_pre_type type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
+    void load(llama_model_loader & ml, const LLM_KV & kv);
-    int max_token_len = 0; // used for optimizing longest token search
+    enum llama_vocab_type     get_type()     const;
+    enum llama_vocab_pre_type get_pre_type() const;
-    std::unordered_map<token, id> token_to_id;
-    std::vector<token_data>       id_to_token;
+    uint32_t n_tokens() const;
+    uint32_t n_token_types() const;
-    std::vector<id>    cache_special_tokens;
-    std::vector<token> cache_token_to_piece; // llama_token_to_piece(special = true);
+    std::string type_name() const;
-    std::map<std::pair<std::string, std::string>, int> bpe_ranks;
+    bool is_normal      (llama_token id) const;
+    bool is_unknown     (llama_token id) const;
+    bool is_control     (llama_token id) const;
+    bool is_byte        (llama_token id) const;
+    bool is_user_defined(llama_token id) const;
+    bool is_unused      (llama_token id) const;
+    bool is_eog         (llama_token id) const;
-    // default LLaMA special tokens
-    // TODO: should we set all of these to LLAMA_TOKEN_NULL?
-    id special_bos_id  = 1;
-    id special_eos_id  = 2;
-    id special_eot_id  = LLAMA_TOKEN_NULL;
-    id special_eom_id  = LLAMA_TOKEN_NULL;
-    id special_unk_id  = 0;
-    id special_sep_id  = LLAMA_TOKEN_NULL;
-    id special_pad_id  = LLAMA_TOKEN_NULL;
-    id special_cls_id  = LLAMA_TOKEN_NULL;
-    id special_mask_id = LLAMA_TOKEN_NULL;
+    uint8_t     token_to_byte(llama_token id) const;
+    llama_token byte_to_token(uint8_t ch)     const;
-    id linefeed_id = 13;
+    llama_token text_to_token(const std::string & text) const;
-    // fim tokens
-    id special_fim_pre_id = LLAMA_TOKEN_NULL;
-    id special_fim_suf_id = LLAMA_TOKEN_NULL;
-    id special_fim_mid_id = LLAMA_TOKEN_NULL;
-    id special_fim_pad_id = LLAMA_TOKEN_NULL;
-    id special_fim_rep_id = LLAMA_TOKEN_NULL; // repo
-    id special_fim_sep_id = LLAMA_TOKEN_NULL; // file separator
+    const token_data & get_token_data(llama_token id) const;
-    // set of all tokens that cause "end of generation"
-    std::set<id> special_eog_ids;
+    const char *     token_get_text (llama_token id) const;
+    float            token_get_score(llama_token id) const;
+    llama_token_attr token_get_attr (llama_token id) const;
-    // tokenizer flags
-    bool tokenizer_add_space_prefix           = false;
-    bool tokenizer_add_bos                    = false;
-    bool tokenizer_add_eos                    = false;
-    bool tokenizer_ignore_merges              = false;
-    bool tokenizer_clean_spaces               = false;  // clean_up_tokenization_spaces
-    bool tokenizer_remove_extra_whitespaces   = false;
-    bool tokenizer_escape_whitespaces         = true;
-    bool tokenizer_treat_whitespace_as_suffix = false;
+    llama_token token_bos() const;
+    llama_token token_eos() const;
+    llama_token token_eot() const;
+    llama_token token_eom() const;
+    llama_token token_unk() const;
+    llama_token token_sep() const;
+    llama_token token_nl () const;
+    llama_token token_pad() const;
-    std::vector<char> precompiled_charsmap;
+    llama_token token_prefix() const;
+    llama_token token_middle() const;
+    llama_token token_suffix() const;
-    llm_tokenizer * tokenizer = nullptr;
+    llama_token token_fim_pre() const;
+    llama_token token_fim_suf() const;
+    llama_token token_fim_mid() const;
+    llama_token token_fim_pad() const;
+    llama_token token_fim_rep() const;
+    llama_token token_fim_sep() const;
-    llama_vocab() = default;
-    ~llama_vocab();
+    bool get_add_space_prefix          () const;
+    bool get_add_bos                   () const;
+    bool get_add_eos                   () const;
+    bool get_ignore_merges             () const;
+    bool get_clean_spaces              () const;
+    bool get_remove_extra_whitespaces  () const;
+    bool get_escape_whitespaces        () const;
+    bool get_treat_whitespace_as_suffix() const;
+    int max_token_len() const;
     int find_bpe_rank(const std::string & token_left, const std::string & token_right) const;
-    void init_tokenizer();
+    int32_t tokenize(
+                   const char * text,
+                      int32_t   text_len,
+                  llama_token * tokens,
+                      int32_t   n_tokens_max,
+                         bool   add_special,
+                         bool   parse_special) const;
+    std::vector<llama_token> tokenize(
+            const std::string & raw_text,
+                         bool   add_special,
+                         bool   parse_special = false) const;
+    // does not write null-terminator to buf
+    int32_t token_to_piece(
+                  llama_token   token,
+                         char * buf,
+                      int32_t   length,
+                      int32_t   lstrip,
+                         bool   special) const;
+    // use cached data
+    const std::string & token_to_piece(llama_token token) const;
+    int32_t detokenize(
+            const llama_token * tokens,
+                      int32_t   n_tokens,
+                         char * text,
+                      int32_t   text_len_max,
+                         bool   remove_special,
+                         bool   unparse_special) const;
+    std::string detokenize(
+            const std::vector<llama_token> & tokens,
+                                      bool   special) const;
+    void print_info() const;
+private:
+    struct impl;
+    std::unique_ptr<impl> pimpl;
 };
-//
-// internal API
-//
-// TODO: rename to llama_tokenize_impl
-// TODO: This should probably be in llama.h
-std::vector<llama_vocab::id> llama_tokenize_internal(
-        const llama_vocab & vocab,
-        std::string raw_text,
-        bool add_special,
-        bool parse_special = false);
-// TODO: move the API below as member functions of llama_vocab
-llama_token llama_byte_to_token_impl(const llama_vocab & vocab, uint8_t ch);
-const char * llama_token_get_text_impl(const struct llama_vocab & vocab, llama_token token);
-float llama_token_get_score_impl(const struct llama_vocab & vocab, llama_token token);
-llama_token_attr llama_token_get_attr_impl(const struct llama_vocab & vocab, llama_token token);
-bool llama_token_is_eog_impl(const struct llama_vocab & vocab, llama_token token);
-bool llama_token_is_control_impl(const struct llama_vocab & vocab, llama_token token);
-llama_token llama_token_bos_impl(const struct llama_vocab & vocab);
-llama_token llama_token_eos_impl(const struct llama_vocab & vocab);
-llama_token llama_token_eot_impl(const struct llama_vocab & vocab);
-llama_token llama_token_eom_impl(const struct llama_vocab & vocab);
-llama_token llama_token_cls_impl(const struct llama_vocab & vocab);
-llama_token llama_token_sep_impl(const struct llama_vocab & vocab);
-llama_token llama_token_nl_impl (const struct llama_vocab & vocab);
-llama_token llama_token_pad_impl(const struct llama_vocab & vocab);
-llama_token llama_token_prefix_impl(const struct llama_vocab & vocab);
-llama_token llama_token_middle_impl(const struct llama_vocab & vocab);
-llama_token llama_token_suffix_impl(const struct llama_vocab & vocab);
-llama_token llama_token_fim_pre_impl(const struct llama_vocab & vocab);
-llama_token llama_token_fim_suf_impl(const struct llama_vocab & vocab);
-llama_token llama_token_fim_mid_impl(const struct llama_vocab & vocab);
-llama_token llama_token_fim_pad_impl(const struct llama_vocab & vocab);
-llama_token llama_token_fim_rep_impl(const struct llama_vocab & vocab);
-llama_token llama_token_fim_sep_impl(const struct llama_vocab & vocab);
-bool llama_add_bos_token_impl(const struct llama_vocab & vocab);
-bool llama_add_eos_token_impl(const struct llama_vocab & vocab);
-int32_t llama_tokenize_impl(
-        const struct llama_vocab & vocab,
-                      const char * text,
-                         int32_t   text_len,
-                     llama_token * tokens,
-                         int32_t   n_tokens_max,
-                            bool   add_special,
-                            bool   parse_special);
-// does not write null-terminator to buf
-int32_t llama_token_to_piece_impl(
-        const struct llama_vocab & vocab,
-                     llama_token   token,
-                            char * buf,
-                         int32_t   length,
-                         int32_t   lstrip,
-                            bool   special);
-// check if token0 is contained as a prefix in token1
-bool llama_token_is_prefix_impl(
-        const struct llama_vocab & vocab,
-                     llama_token   token0,
-                     llama_token   token1);
-int32_t llama_detokenize_impl(
-        const struct llama_vocab & vocab,
-               const llama_token * tokens,
-                         int32_t   n_tokens,
-                            char * text,
-                         int32_t   text_len_max,
-                            bool   remove_special,
-                            bool   unparse_special);
-std::string llama_detokenize(
-        const struct llama_vocab & vocab,
-  const std::vector<llama_token> & tokens,
-                            bool   special);