@novastera-oss/llamarn 0.0.1-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INTERFACE.md +389 -0
- package/LICENSE +201 -0
- package/README.md +235 -0
- package/RNLlamaCpp.podspec +69 -0
- package/android/CMakeLists.txt +107 -0
- package/android/build.gradle +111 -0
- package/android/generated/java/com/novastera/llamarn/NativeRNLlamaCppSpec.java +47 -0
- package/android/generated/jni/CMakeLists.txt +36 -0
- package/android/generated/jni/RNLlamaCppSpec-generated.cpp +44 -0
- package/android/generated/jni/RNLlamaCppSpec.h +31 -0
- package/android/generated/jni/react/renderer/components/RNLlamaCppSpec/RNLlamaCppSpecJSI-generated.cpp +42 -0
- package/android/generated/jni/react/renderer/components/RNLlamaCppSpec/RNLlamaCppSpecJSI.h +336 -0
- package/android/gradle.properties +5 -0
- package/android/src/main/AndroidManifest.xml +3 -0
- package/android/src/main/AndroidManifestNew.xml +2 -0
- package/android/src/main/cpp/include/llama-cpp.h +30 -0
- package/android/src/main/cpp/include/llama.h +1440 -0
- package/android/src/main/java/com/novastera/llamarn/RNLlamaCppPackage.kt +21 -0
- package/android/src/main/jniLibs/arm64-v8a/libOpenCL.so +0 -0
- package/android/src/main/jniLibs/arm64-v8a/libggml-base.so +0 -0
- package/android/src/main/jniLibs/arm64-v8a/libggml-cpu.so +0 -0
- package/android/src/main/jniLibs/arm64-v8a/libggml.so +0 -0
- package/android/src/main/jniLibs/arm64-v8a/libllama.so +0 -0
- package/android/src/main/jniLibs/x86_64/libOpenCL.so +0 -0
- package/android/src/main/jniLibs/x86_64/libggml-base.so +0 -0
- package/android/src/main/jniLibs/x86_64/libggml-cpu.so +0 -0
- package/android/src/main/jniLibs/x86_64/libggml.so +0 -0
- package/android/src/main/jniLibs/x86_64/libllama.so +0 -0
- package/cpp/LlamaCppModel.cpp +984 -0
- package/cpp/LlamaCppModel.h +162 -0
- package/cpp/PureCppImpl.cpp +308 -0
- package/cpp/PureCppImpl.h +59 -0
- package/cpp/SystemUtils.cpp +180 -0
- package/cpp/SystemUtils.h +74 -0
- package/cpp/build-info.cpp +4 -0
- package/cpp/llama.cpp/AUTHORS +1106 -0
- package/cpp/llama.cpp/CMakeLists.txt +254 -0
- package/cpp/llama.cpp/CMakePresets.json +84 -0
- package/cpp/llama.cpp/CODEOWNERS +11 -0
- package/cpp/llama.cpp/CONTRIBUTING.md +127 -0
- package/cpp/llama.cpp/LICENSE +21 -0
- package/cpp/llama.cpp/Makefile +1608 -0
- package/cpp/llama.cpp/README.md +575 -0
- package/cpp/llama.cpp/SECURITY.md +68 -0
- package/cpp/llama.cpp/build-xcframework.sh +540 -0
- package/cpp/llama.cpp/cmake/arm64-apple-clang.cmake +16 -0
- package/cpp/llama.cpp/cmake/arm64-windows-llvm.cmake +16 -0
- package/cpp/llama.cpp/cmake/build-info.cmake +64 -0
- package/cpp/llama.cpp/cmake/common.cmake +35 -0
- package/cpp/llama.cpp/cmake/git-vars.cmake +22 -0
- package/cpp/llama.cpp/cmake/llama-config.cmake.in +30 -0
- package/cpp/llama.cpp/cmake/llama.pc.in +10 -0
- package/cpp/llama.cpp/cmake/x64-windows-llvm.cmake +5 -0
- package/cpp/llama.cpp/common/CMakeLists.txt +170 -0
- package/cpp/llama.cpp/common/arg.cpp +3337 -0
- package/cpp/llama.cpp/common/arg.h +89 -0
- package/cpp/llama.cpp/common/base64.hpp +392 -0
- package/cpp/llama.cpp/common/build-info.cpp.in +4 -0
- package/cpp/llama.cpp/common/chat.cpp +1781 -0
- package/cpp/llama.cpp/common/chat.h +135 -0
- package/cpp/llama.cpp/common/cmake/build-info-gen-cpp.cmake +24 -0
- package/cpp/llama.cpp/common/common.cpp +1567 -0
- package/cpp/llama.cpp/common/common.h +668 -0
- package/cpp/llama.cpp/common/console.cpp +504 -0
- package/cpp/llama.cpp/common/console.h +19 -0
- package/cpp/llama.cpp/common/json-schema-to-grammar.cpp +1027 -0
- package/cpp/llama.cpp/common/json-schema-to-grammar.h +21 -0
- package/cpp/llama.cpp/common/json.hpp +24766 -0
- package/cpp/llama.cpp/common/llguidance.cpp +254 -0
- package/cpp/llama.cpp/common/log.cpp +393 -0
- package/cpp/llama.cpp/common/log.h +103 -0
- package/cpp/llama.cpp/common/minja/chat-template.hpp +537 -0
- package/cpp/llama.cpp/common/minja/minja.hpp +2941 -0
- package/cpp/llama.cpp/common/ngram-cache.cpp +286 -0
- package/cpp/llama.cpp/common/ngram-cache.h +101 -0
- package/cpp/llama.cpp/common/sampling.cpp +580 -0
- package/cpp/llama.cpp/common/sampling.h +107 -0
- package/cpp/llama.cpp/common/speculative.cpp +278 -0
- package/cpp/llama.cpp/common/speculative.h +28 -0
- package/cpp/llama.cpp/common/stb_image.h +7988 -0
- package/cpp/llama.cpp/convert_hf_to_gguf.py +6195 -0
- package/cpp/llama.cpp/convert_hf_to_gguf_update.py +393 -0
- package/cpp/llama.cpp/convert_llama_ggml_to_gguf.py +450 -0
- package/cpp/llama.cpp/convert_lora_to_gguf.py +461 -0
- package/cpp/llama.cpp/flake.lock +58 -0
- package/cpp/llama.cpp/flake.nix +185 -0
- package/cpp/llama.cpp/ggml/CMakeLists.txt +388 -0
- package/cpp/llama.cpp/ggml/cmake/GitVars.cmake +22 -0
- package/cpp/llama.cpp/ggml/cmake/common.cmake +26 -0
- package/cpp/llama.cpp/ggml/cmake/ggml-config.cmake.in +152 -0
- package/cpp/llama.cpp/ggml/include/ggml-alloc.h +76 -0
- package/cpp/llama.cpp/ggml/include/ggml-backend.h +354 -0
- package/cpp/llama.cpp/ggml/include/ggml-blas.h +25 -0
- package/cpp/llama.cpp/ggml/include/ggml-cann.h +123 -0
- package/cpp/llama.cpp/ggml/include/ggml-cpp.h +39 -0
- package/cpp/llama.cpp/ggml/include/ggml-cpu.h +143 -0
- package/cpp/llama.cpp/ggml/include/ggml-cuda.h +47 -0
- package/cpp/llama.cpp/ggml/include/ggml-kompute.h +50 -0
- package/cpp/llama.cpp/ggml/include/ggml-metal.h +66 -0
- package/cpp/llama.cpp/ggml/include/ggml-opencl.h +26 -0
- package/cpp/llama.cpp/ggml/include/ggml-opt.h +216 -0
- package/cpp/llama.cpp/ggml/include/ggml-rpc.h +33 -0
- package/cpp/llama.cpp/ggml/include/ggml-sycl.h +49 -0
- package/cpp/llama.cpp/ggml/include/ggml-vulkan.h +29 -0
- package/cpp/llama.cpp/ggml/include/ggml.h +2192 -0
- package/cpp/llama.cpp/ggml/include/gguf.h +202 -0
- package/cpp/llama.cpp/ggml/src/CMakeLists.txt +345 -0
- package/cpp/llama.cpp/ggml/src/ggml-alloc.c +1042 -0
- package/cpp/llama.cpp/ggml/src/ggml-backend-impl.h +255 -0
- package/cpp/llama.cpp/ggml/src/ggml-backend-reg.cpp +586 -0
- package/cpp/llama.cpp/ggml/src/ggml-backend.cpp +2008 -0
- package/cpp/llama.cpp/ggml/src/ggml-blas/CMakeLists.txt +87 -0
- package/cpp/llama.cpp/ggml/src/ggml-blas/ggml-blas.cpp +517 -0
- package/cpp/llama.cpp/ggml/src/ggml-cann/CMakeLists.txt +74 -0
- package/cpp/llama.cpp/ggml/src/ggml-cann/Doxyfile +2579 -0
- package/cpp/llama.cpp/ggml/src/ggml-cann/acl_tensor.cpp +179 -0
- package/cpp/llama.cpp/ggml/src/ggml-cann/acl_tensor.h +258 -0
- package/cpp/llama.cpp/ggml/src/ggml-cann/aclnn_ops.cpp +2589 -0
- package/cpp/llama.cpp/ggml/src/ggml-cann/aclnn_ops.h +1083 -0
- package/cpp/llama.cpp/ggml/src/ggml-cann/common.h +420 -0
- package/cpp/llama.cpp/ggml/src/ggml-cann/ggml-cann.cpp +2554 -0
- package/cpp/llama.cpp/ggml/src/ggml-common.h +1857 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt +495 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/amx/amx.cpp +221 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/amx/amx.h +8 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/amx/common.h +91 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/amx/mmq.cpp +2511 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/amx/mmq.h +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/binary-ops.cpp +158 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/binary-ops.h +16 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/common.h +72 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/cpu-feats-x86.cpp +327 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +6431 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-aarch64.h +8 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-hbm.cpp +55 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-hbm.h +8 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-impl.h +512 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-quants.c +13131 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-quants.h +63 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-traits.cpp +36 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-traits.h +38 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +3492 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp +671 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/kleidiai/kernels.cpp +254 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/kleidiai/kernels.h +60 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +287 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/llamafile/sgemm.cpp +3544 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/llamafile/sgemm.h +14 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/ops.cpp +8796 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/ops.h +110 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/simd-mappings.h +892 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/unary-ops.cpp +186 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/unary-ops.h +28 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/vec.cpp +252 -0
- package/cpp/llama.cpp/ggml/src/ggml-cpu/vec.h +802 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/CMakeLists.txt +184 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/acc.cu +47 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/acc.cuh +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/arange.cu +34 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/arange.cuh +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/argmax.cu +91 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/argmax.cuh +3 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/argsort.cu +104 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/argsort.cuh +3 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/binbcast.cu +363 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/binbcast.cuh +9 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/clamp.cu +45 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/clamp.cuh +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/common.cuh +828 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/concat.cu +221 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/concat.cuh +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/conv-transpose-1d.cu +89 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/convert.cu +730 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/convert.cuh +26 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/count-equal.cu +64 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/cpy.cu +695 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/cpy.cuh +11 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/cross-entropy-loss.cu +189 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/dequantize.cuh +103 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/diagmask.cu +40 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +873 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1269 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-tile-f16.cu +357 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-tile-f16.cuh +3 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-tile-f32.cu +365 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-tile-f32.cuh +3 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-vec-f16.cuh +437 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-vec-f32.cuh +428 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-wmma-f16.cu +634 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +3 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn.cu +345 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn.cuh +3 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/getrows.cu +275 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/getrows.cuh +15 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +3501 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/gla.cu +93 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/gla.cuh +3 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/im2col.cu +103 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/im2col.cuh +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/mma.cuh +396 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/mmq.cu +322 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/mmq.cuh +3217 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/mmv.cu +336 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/mmv.cuh +12 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/mmvq.cu +595 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/mmvq.cuh +12 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/norm.cu +458 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/norm.cuh +11 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/out-prod.cu +68 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/pad.cu +49 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/pad.cuh +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/pool2d.cu +94 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/quantize.cu +189 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/quantize.cuh +27 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/rope.cu +456 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/rope.cuh +7 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/scale.cu +31 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/scale.cuh +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/softmax.cu +283 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/softmax.cuh +7 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/ssm-conv.cu +148 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/ssm-scan.cu +153 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/sum.cu +45 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/sum.cuh +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/sumrows.cu +39 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/sumrows.cuh +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +78 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/tsembd.cu +47 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/unary.cu +279 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/unary.cuh +57 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/upscale.cu +51 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/upscale.cuh +5 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/vecdotq.cuh +1135 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/vendors/cuda.h +15 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/vendors/hip.h +243 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/vendors/musa.h +140 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/wkv.cu +199 -0
- package/cpp/llama.cpp/ggml/src/ggml-cuda/wkv.cuh +7 -0
- package/cpp/llama.cpp/ggml/src/ggml-hip/CMakeLists.txt +131 -0
- package/cpp/llama.cpp/ggml/src/ggml-impl.h +601 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/CMakeLists.txt +166 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/ggml-kompute.cpp +2251 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/common.comp +112 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_add.comp +58 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_addrow.comp +25 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f16.comp +52 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f32.comp +52 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f16.comp +52 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f32.comp +52 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_diagmask.comp +30 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_gelu.comp +22 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_getrows.comp +17 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f16.comp +31 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f32.comp +31 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_0.comp +38 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_1.comp +39 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q6_k.comp +44 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul.comp +52 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_f16.comp +69 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_mat_f32.comp +51 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_0.comp +33 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_1.comp +35 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_k.comp +140 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q6_k.comp +106 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q8_0.comp +73 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n.comp +52 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n_pre.comp +28 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_norm.comp +84 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_relu.comp +21 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_rmsnorm.comp +53 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f16.comp +52 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f32.comp +52 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f16.comp +52 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f32.comp +52 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_scale.comp +19 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_scale_8.comp +23 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_silu.comp +22 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/op_softmax.comp +72 -0
- package/cpp/llama.cpp/ggml/src/ggml-kompute/kompute-shaders/rope_common.comp +71 -0
- package/cpp/llama.cpp/ggml/src/ggml-metal/CMakeLists.txt +120 -0
- package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal-impl.h +618 -0
- package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal.m +5916 -0
- package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal.metal +6891 -0
- package/cpp/llama.cpp/ggml/src/ggml-musa/CMakeLists.txt +107 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/CMakeLists.txt +96 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/ggml-opencl.cpp +4966 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/add.cl +83 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/cpy.cl +184 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/cvt.cl +118 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/gelu.cl +62 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl +268 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl +274 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/get_rows.cl +163 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul.cl +79 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl +139 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +118 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +118 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +94 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +84 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q6_k.cl +190 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/norm.cl +81 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/rms_norm.cl +96 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/rope.cl +721 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/scale.cl +16 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +87 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +87 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/softmax_f16.cl +86 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/softmax_f32.cl +86 -0
- package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/transpose.cl +84 -0
- package/cpp/llama.cpp/ggml/src/ggml-opt.cpp +854 -0
- package/cpp/llama.cpp/ggml/src/ggml-quants.c +5232 -0
- package/cpp/llama.cpp/ggml/src/ggml-quants.h +100 -0
- package/cpp/llama.cpp/ggml/src/ggml-rpc/CMakeLists.txt +9 -0
- package/cpp/llama.cpp/ggml/src/ggml-rpc/ggml-rpc.cpp +1813 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/CMakeLists.txt +183 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/backend.hpp +37 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/binbcast.cpp +350 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/binbcast.hpp +39 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/common.cpp +83 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/common.hpp +493 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/concat.cpp +197 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/concat.hpp +20 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/conv.cpp +100 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/conv.hpp +20 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/convert.cpp +596 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/convert.hpp +34 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/cpy.cpp +701 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/cpy.hpp +11 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/dequantize.hpp +753 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/dmmv.cpp +1154 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/dmmv.hpp +27 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/dpct/helper.hpp +2957 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/element_wise.cpp +1559 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/element_wise.hpp +75 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/gemm.hpp +70 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/getrows.cpp +311 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/getrows.hpp +20 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/ggml-sycl.cpp +4302 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/gla.cpp +105 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/gla.hpp +8 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/im2col.cpp +136 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/im2col.hpp +21 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/mmq.cpp +3030 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/mmq.hpp +33 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/mmvq.cpp +1081 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/mmvq.hpp +27 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/norm.cpp +474 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/norm.hpp +26 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/outprod.cpp +46 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/outprod.hpp +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/presets.hpp +74 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/quants.hpp +61 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/rope.cpp +362 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/rope.hpp +20 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/softmax.cpp +264 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/softmax.hpp +20 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/sycl_hw.cpp +13 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/sycl_hw.hpp +23 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/tsembd.cpp +73 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/tsembd.hpp +20 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/vecdotq.hpp +1189 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/wkv.cpp +305 -0
- package/cpp/llama.cpp/ggml/src/ggml-sycl/wkv.hpp +10 -0
- package/cpp/llama.cpp/ggml/src/ggml-threading.cpp +12 -0
- package/cpp/llama.cpp/ggml/src/ggml-threading.h +14 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/CMakeLists.txt +202 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp +10502 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +22 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +29 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +29 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +51 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +69 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp +17 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +49 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +23 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +242 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/cos.comp +17 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp +462 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +699 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.comp +13 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +48 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +39 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +49 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +31 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +483 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +383 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +59 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/gelu.comp +25 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/gelu_quick.comp +23 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.comp +64 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.comp +9 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.comp +76 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +33 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +41 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +100 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp +22 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +169 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.comp +118 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +82 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +79 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +87 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +90 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +118 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +154 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +130 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +136 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +167 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +868 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +441 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +442 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.comp +99 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +28 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +77 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/relu.comp +21 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +52 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp +58 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +60 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +43 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +43 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +47 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp +20 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/silu.comp +22 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/sin.comp +17 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +173 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +50 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/square.comp +17 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +37 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/tanh.comp +20 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/test_bfloat16_support.comp +7 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat2_support.comp +7 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat_support.comp +7 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/test_integer_dot_support.comp +7 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +41 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +1373 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +36 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +740 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
- package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
- package/cpp/llama.cpp/ggml/src/ggml.c +6499 -0
- package/cpp/llama.cpp/ggml/src/gguf.cpp +1330 -0
- package/cpp/llama.cpp/gguf-py/LICENSE +21 -0
- package/cpp/llama.cpp/gguf-py/README.md +99 -0
- package/cpp/llama.cpp/gguf-py/examples/reader.py +49 -0
- package/cpp/llama.cpp/gguf-py/examples/writer.py +39 -0
- package/cpp/llama.cpp/gguf-py/gguf/__init__.py +9 -0
- package/cpp/llama.cpp/gguf-py/gguf/constants.py +2296 -0
- package/cpp/llama.cpp/gguf-py/gguf/gguf.py +15 -0
- package/cpp/llama.cpp/gguf-py/gguf/gguf_reader.py +367 -0
- package/cpp/llama.cpp/gguf-py/gguf/gguf_writer.py +1041 -0
- package/cpp/llama.cpp/gguf-py/gguf/lazy.py +223 -0
- package/cpp/llama.cpp/gguf-py/gguf/metadata.py +642 -0
- package/cpp/llama.cpp/gguf-py/gguf/py.typed +0 -0
- package/cpp/llama.cpp/gguf-py/gguf/quants.py +1269 -0
- package/cpp/llama.cpp/gguf-py/gguf/scripts/gguf_convert_endian.py +182 -0
- package/cpp/llama.cpp/gguf-py/gguf/scripts/gguf_dump.py +454 -0
- package/cpp/llama.cpp/gguf-py/gguf/scripts/gguf_editor_gui.py +1610 -0
- package/cpp/llama.cpp/gguf-py/gguf/scripts/gguf_hash.py +102 -0
- package/cpp/llama.cpp/gguf-py/gguf/scripts/gguf_new_metadata.py +207 -0
- package/cpp/llama.cpp/gguf-py/gguf/scripts/gguf_set_metadata.py +95 -0
- package/cpp/llama.cpp/gguf-py/gguf/tensor_mapping.py +1172 -0
- package/cpp/llama.cpp/gguf-py/gguf/utility.py +264 -0
- package/cpp/llama.cpp/gguf-py/gguf/vocab.py +492 -0
- package/cpp/llama.cpp/gguf-py/pyproject.toml +43 -0
- package/cpp/llama.cpp/gguf-py/tests/__init__.py +1 -0
- package/cpp/llama.cpp/gguf-py/tests/test_metadata.py +238 -0
- package/cpp/llama.cpp/gguf-py/tests/test_quants.py +238 -0
- package/cpp/llama.cpp/grammars/README.md +382 -0
- package/cpp/llama.cpp/grammars/arithmetic.gbnf +6 -0
- package/cpp/llama.cpp/grammars/c.gbnf +42 -0
- package/cpp/llama.cpp/grammars/chess.gbnf +13 -0
- package/cpp/llama.cpp/grammars/english.gbnf +6 -0
- package/cpp/llama.cpp/grammars/japanese.gbnf +7 -0
- package/cpp/llama.cpp/grammars/json.gbnf +25 -0
- package/cpp/llama.cpp/grammars/json_arr.gbnf +34 -0
- package/cpp/llama.cpp/grammars/list.gbnf +4 -0
- package/cpp/llama.cpp/include/llama-cpp.h +30 -0
- package/cpp/llama.cpp/include/llama.h +1440 -0
- package/cpp/llama.cpp/licenses/LICENSE-curl +9 -0
- package/cpp/llama.cpp/licenses/LICENSE-httplib +21 -0
- package/cpp/llama.cpp/licenses/LICENSE-jsonhpp +21 -0
- package/cpp/llama.cpp/licenses/LICENSE-linenoise +26 -0
- package/cpp/llama.cpp/media/llama0-banner.png +0 -0
- package/cpp/llama.cpp/media/llama0-logo.png +0 -0
- package/cpp/llama.cpp/media/llama1-banner.png +0 -0
- package/cpp/llama.cpp/media/llama1-logo.png +0 -0
- package/cpp/llama.cpp/media/llama1-logo.svg +34 -0
- package/cpp/llama.cpp/media/matmul.png +0 -0
- package/cpp/llama.cpp/media/matmul.svg +1238 -0
- package/cpp/llama.cpp/models/ggml-vocab-aquila.gguf +0 -0
- package/cpp/llama.cpp/models/ggml-vocab-baichuan.gguf +0 -0
- package/cpp/llama.cpp/models/ggml-vocab-bert-bge.gguf +0 -0
- package/cpp/llama.cpp/models/ggml-vocab-bert-bge.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-bert-bge.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-chameleon.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-chameleon.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-command-r.gguf +0 -0
- package/cpp/llama.cpp/models/ggml-vocab-command-r.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-command-r.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-deepseek-coder.gguf +0 -0
- package/cpp/llama.cpp/models/ggml-vocab-deepseek-coder.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-deepseek-coder.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-deepseek-llm.gguf +0 -0
- package/cpp/llama.cpp/models/ggml-vocab-deepseek-llm.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-deepseek-llm.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-deepseek-r1-qwen.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-deepseek-r1-qwen.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-falcon.gguf +0 -0
- package/cpp/llama.cpp/models/ggml-vocab-falcon.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-falcon.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-gpt-2.gguf +0 -0
- package/cpp/llama.cpp/models/ggml-vocab-gpt-2.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-gpt-2.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-gpt-4o.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-gpt-4o.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-gpt-neox.gguf +0 -0
- package/cpp/llama.cpp/models/ggml-vocab-llama-bpe.gguf +0 -0
- package/cpp/llama.cpp/models/ggml-vocab-llama-bpe.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-llama-bpe.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-llama-spm.gguf +0 -0
- package/cpp/llama.cpp/models/ggml-vocab-llama-spm.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-llama-spm.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-llama4.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-llama4.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-mpt.gguf +0 -0
- package/cpp/llama.cpp/models/ggml-vocab-mpt.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-mpt.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-phi-3.gguf +0 -0
- package/cpp/llama.cpp/models/ggml-vocab-phi-3.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-phi-3.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-pixtral.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-pixtral.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-qwen2.gguf +0 -0
- package/cpp/llama.cpp/models/ggml-vocab-qwen2.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-qwen2.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-refact.gguf +0 -0
- package/cpp/llama.cpp/models/ggml-vocab-refact.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-refact.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-roberta-bpe.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-roberta-bpe.gguf.out +46 -0
- package/cpp/llama.cpp/models/ggml-vocab-starcoder.gguf +0 -0
- package/cpp/llama.cpp/models/ggml-vocab-starcoder.gguf.inp +112 -0
- package/cpp/llama.cpp/models/ggml-vocab-starcoder.gguf.out +46 -0
- package/cpp/llama.cpp/models/templates/CohereForAI-c4ai-command-r-plus-tool_use.jinja +202 -0
- package/cpp/llama.cpp/models/templates/CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja +156 -0
- package/cpp/llama.cpp/models/templates/NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja +152 -0
- package/cpp/llama.cpp/models/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja +152 -0
- package/cpp/llama.cpp/models/templates/Qwen-Qwen2.5-7B-Instruct.jinja +54 -0
- package/cpp/llama.cpp/models/templates/README.md +22 -0
- package/cpp/llama.cpp/models/templates/deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja +1 -0
- package/cpp/llama.cpp/models/templates/deepseek-ai-DeepSeek-R1-Distill-Qwen-32B.jinja +1 -0
- package/cpp/llama.cpp/models/templates/fireworks-ai-llama-3-firefunction-v2.jinja +57 -0
- package/cpp/llama.cpp/models/templates/google-gemma-2-2b-it.jinja +4 -0
- package/cpp/llama.cpp/models/templates/llama-cpp-deepseek-r1.jinja +76 -0
- package/cpp/llama.cpp/models/templates/meetkai-functionary-medium-v3.1.jinja +58 -0
- package/cpp/llama.cpp/models/templates/meetkai-functionary-medium-v3.2.jinja +287 -0
- package/cpp/llama.cpp/models/templates/meta-llama-Llama-3.1-8B-Instruct.jinja +109 -0
- package/cpp/llama.cpp/models/templates/meta-llama-Llama-3.2-3B-Instruct.jinja +93 -0
- package/cpp/llama.cpp/models/templates/meta-llama-Llama-3.3-70B-Instruct.jinja +109 -0
- package/cpp/llama.cpp/models/templates/microsoft-Phi-3.5-mini-instruct.jinja +8 -0
- package/cpp/llama.cpp/models/templates/mistralai-Mistral-Nemo-Instruct-2407.jinja +87 -0
- package/cpp/llama.cpp/mypy.ini +7 -0
- package/cpp/llama.cpp/pocs/CMakeLists.txt +14 -0
- package/cpp/llama.cpp/pocs/vdot/CMakeLists.txt +9 -0
- package/cpp/llama.cpp/pocs/vdot/q8dot.cpp +173 -0
- package/cpp/llama.cpp/pocs/vdot/vdot.cpp +311 -0
- package/cpp/llama.cpp/poetry.lock +1197 -0
- package/cpp/llama.cpp/prompts/LLM-questions.txt +49 -0
- package/cpp/llama.cpp/prompts/alpaca.txt +1 -0
- package/cpp/llama.cpp/prompts/assistant.txt +31 -0
- package/cpp/llama.cpp/prompts/chat-with-baichuan.txt +4 -0
- package/cpp/llama.cpp/prompts/chat-with-bob.txt +7 -0
- package/cpp/llama.cpp/prompts/chat-with-qwen.txt +1 -0
- package/cpp/llama.cpp/prompts/chat-with-vicuna-v0.txt +7 -0
- package/cpp/llama.cpp/prompts/chat-with-vicuna-v1.txt +7 -0
- package/cpp/llama.cpp/prompts/chat.txt +28 -0
- package/cpp/llama.cpp/prompts/dan-modified.txt +1 -0
- package/cpp/llama.cpp/prompts/dan.txt +1 -0
- package/cpp/llama.cpp/prompts/mnemonics.txt +93 -0
- package/cpp/llama.cpp/prompts/parallel-questions.txt +43 -0
- package/cpp/llama.cpp/prompts/reason-act.txt +18 -0
- package/cpp/llama.cpp/pyproject.toml +45 -0
- package/cpp/llama.cpp/pyrightconfig.json +22 -0
- package/cpp/llama.cpp/requirements/requirements-all.txt +15 -0
- package/cpp/llama.cpp/requirements/requirements-compare-llama-bench.txt +2 -0
- package/cpp/llama.cpp/requirements/requirements-convert_hf_to_gguf.txt +3 -0
- package/cpp/llama.cpp/requirements/requirements-convert_hf_to_gguf_update.txt +3 -0
- package/cpp/llama.cpp/requirements/requirements-convert_legacy_llama.txt +5 -0
- package/cpp/llama.cpp/requirements/requirements-convert_llama_ggml_to_gguf.txt +1 -0
- package/cpp/llama.cpp/requirements/requirements-convert_lora_to_gguf.txt +2 -0
- package/cpp/llama.cpp/requirements/requirements-gguf_editor_gui.txt +3 -0
- package/cpp/llama.cpp/requirements/requirements-pydantic.txt +3 -0
- package/cpp/llama.cpp/requirements/requirements-test-tokenizer-random.txt +1 -0
- package/cpp/llama.cpp/requirements/requirements-tool_bench.txt +12 -0
- package/cpp/llama.cpp/requirements.txt +13 -0
- package/cpp/llama.cpp/src/CMakeLists.txt +45 -0
- package/cpp/llama.cpp/src/llama-adapter.cpp +388 -0
- package/cpp/llama.cpp/src/llama-adapter.h +76 -0
- package/cpp/llama.cpp/src/llama-arch.cpp +1743 -0
- package/cpp/llama.cpp/src/llama-arch.h +437 -0
- package/cpp/llama.cpp/src/llama-batch.cpp +372 -0
- package/cpp/llama.cpp/src/llama-batch.h +89 -0
- package/cpp/llama.cpp/src/llama-chat.cpp +663 -0
- package/cpp/llama.cpp/src/llama-chat.h +58 -0
- package/cpp/llama.cpp/src/llama-context.cpp +2459 -0
- package/cpp/llama.cpp/src/llama-context.h +246 -0
- package/cpp/llama.cpp/src/llama-cparams.cpp +1 -0
- package/cpp/llama.cpp/src/llama-cparams.h +39 -0
- package/cpp/llama.cpp/src/llama-grammar.cpp +1219 -0
- package/cpp/llama.cpp/src/llama-grammar.h +173 -0
- package/cpp/llama.cpp/src/llama-graph.cpp +1713 -0
- package/cpp/llama.cpp/src/llama-graph.h +595 -0
- package/cpp/llama.cpp/src/llama-hparams.cpp +79 -0
- package/cpp/llama.cpp/src/llama-hparams.h +161 -0
- package/cpp/llama.cpp/src/llama-impl.cpp +167 -0
- package/cpp/llama.cpp/src/llama-impl.h +61 -0
- package/cpp/llama.cpp/src/llama-io.cpp +15 -0
- package/cpp/llama.cpp/src/llama-io.h +35 -0
- package/cpp/llama.cpp/src/llama-kv-cache.cpp +2486 -0
- package/cpp/llama.cpp/src/llama-kv-cache.h +405 -0
- package/cpp/llama.cpp/src/llama-memory.cpp +1 -0
- package/cpp/llama.cpp/src/llama-memory.h +31 -0
- package/cpp/llama.cpp/src/llama-mmap.cpp +600 -0
- package/cpp/llama.cpp/src/llama-mmap.h +68 -0
- package/cpp/llama.cpp/src/llama-model-loader.cpp +1133 -0
- package/cpp/llama.cpp/src/llama-model-loader.h +169 -0
- package/cpp/llama.cpp/src/llama-model.cpp +13453 -0
- package/cpp/llama.cpp/src/llama-model.h +420 -0
- package/cpp/llama.cpp/src/llama-quant.cpp +964 -0
- package/cpp/llama.cpp/src/llama-quant.h +1 -0
- package/cpp/llama.cpp/src/llama-sampling.cpp +2575 -0
- package/cpp/llama.cpp/src/llama-sampling.h +32 -0
- package/cpp/llama.cpp/src/llama-vocab.cpp +3313 -0
- package/cpp/llama.cpp/src/llama-vocab.h +125 -0
- package/cpp/llama.cpp/src/llama.cpp +340 -0
- package/cpp/llama.cpp/src/unicode-data.cpp +7034 -0
- package/cpp/llama.cpp/src/unicode-data.h +20 -0
- package/cpp/llama.cpp/src/unicode.cpp +849 -0
- package/cpp/llama.cpp/src/unicode.h +66 -0
- package/cpp/rn-completion.cpp +431 -0
- package/cpp/rn-llama.hpp +60 -0
- package/cpp/rn-utils.hpp +331 -0
- package/ios/OnLoad.mm +22 -0
- package/ios/generated/RNLlamaCppSpec/RNLlamaCppSpec-generated.mm +64 -0
- package/ios/generated/RNLlamaCppSpec/RNLlamaCppSpec.h +251 -0
- package/ios/generated/RNLlamaCppSpecJSI-generated.cpp +42 -0
- package/ios/generated/RNLlamaCppSpecJSI.h +336 -0
- package/ios/include/chat.h +135 -0
- package/ios/include/common/base64.hpp +392 -0
- package/ios/include/common/json.hpp +24766 -0
- package/ios/include/common/minja/chat-template.hpp +537 -0
- package/ios/include/common/minja/minja.hpp +2941 -0
- package/ios/include/common.h +668 -0
- package/ios/include/json-schema-to-grammar.h +21 -0
- package/ios/include/llama-cpp.h +30 -0
- package/ios/include/llama.h +1440 -0
- package/ios/include/log.h +103 -0
- package/ios/include/ngram-cache.h +101 -0
- package/ios/include/sampling.h +107 -0
- package/ios/include/speculative.h +28 -0
- package/ios/libs/llama.xcframework/Info.plist +135 -0
- package/ios/libs/llama.xcframework/ios-arm64/dSYMs/llama.dSYM/Contents/Info.plist +20 -0
- package/ios/libs/llama.xcframework/ios-arm64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
- package/ios/libs/llama.xcframework/ios-arm64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +4492 -0
- package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/ggml-alloc.h +76 -0
- package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/ggml-backend.h +354 -0
- package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/ggml-blas.h +25 -0
- package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/ggml-cpu.h +143 -0
- package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/ggml-metal.h +66 -0
- package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/ggml.h +2192 -0
- package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/gguf.h +202 -0
- package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/llama.h +1440 -0
- package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Info.plist +36 -0
- package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Modules/module.modulemap +17 -0
- package/ios/libs/llama.xcframework/ios-arm64/llama.framework/llama +0 -0
- package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Info.plist +20 -0
- package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
- package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +4513 -0
- package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +3440 -0
- package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml-alloc.h +76 -0
- package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml-backend.h +354 -0
- package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml-blas.h +25 -0
- package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml-cpu.h +143 -0
- package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml-metal.h +66 -0
- package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml.h +2192 -0
- package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/gguf.h +202 -0
- package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/llama.h +1440 -0
- package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Info.plist +36 -0
- package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Modules/module.modulemap +17 -0
- package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/llama +0 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Info.plist +20 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +4513 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +3442 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml-alloc.h +76 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml-backend.h +354 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml-blas.h +25 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml-cpu.h +143 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml-metal.h +66 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml.h +2192 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/gguf.h +202 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/llama.h +1440 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Modules/module.modulemap +17 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Resources/Info.plist +32 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-alloc.h +76 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-backend.h +354 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-blas.h +25 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-cpu.h +143 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-metal.h +66 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml.h +2192 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/gguf.h +202 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/llama.h +1440 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Modules/module.modulemap +17 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Resources/Info.plist +32 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/llama +0 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml-alloc.h +76 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml-backend.h +354 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml-blas.h +25 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml-cpu.h +143 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml-metal.h +66 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml.h +2192 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/gguf.h +202 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/llama.h +1440 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Modules/module.modulemap +17 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Resources/Info.plist +32 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/llama +0 -0
- package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/llama +0 -0
- package/ios/libs/llama.xcframework/tvos-arm64/dSYMs/llama.dSYM/Contents/Info.plist +20 -0
- package/ios/libs/llama.xcframework/tvos-arm64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
- package/ios/libs/llama.xcframework/tvos-arm64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +4492 -0
- package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml-alloc.h +76 -0
- package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml-backend.h +354 -0
- package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml-blas.h +25 -0
- package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml-cpu.h +143 -0
- package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml-metal.h +66 -0
- package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml.h +2192 -0
- package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/gguf.h +202 -0
- package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/llama.h +1440 -0
- package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Info.plist +35 -0
- package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Modules/module.modulemap +17 -0
- package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/llama +0 -0
- package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Info.plist +20 -0
- package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
- package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +4513 -0
- package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +3440 -0
- package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml-alloc.h +76 -0
- package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml-backend.h +354 -0
- package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml-blas.h +25 -0
- package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml-cpu.h +143 -0
- package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml-metal.h +66 -0
- package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml.h +2192 -0
- package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/gguf.h +202 -0
- package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/llama.h +1440 -0
- package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Info.plist +35 -0
- package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Modules/module.modulemap +17 -0
- package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/llama +0 -0
- package/ios/libs/llama.xcframework/xros-arm64/dSYMs/llama.dSYM/Contents/Info.plist +20 -0
- package/ios/libs/llama.xcframework/xros-arm64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
- package/ios/libs/llama.xcframework/xros-arm64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +4528 -0
- package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml-alloc.h +76 -0
- package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml-backend.h +354 -0
- package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml-blas.h +25 -0
- package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml-cpu.h +143 -0
- package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml-metal.h +66 -0
- package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml.h +2192 -0
- package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/gguf.h +202 -0
- package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/llama.h +1440 -0
- package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Info.plist +32 -0
- package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Modules/module.modulemap +17 -0
- package/ios/libs/llama.xcframework/xros-arm64/llama.framework/llama +0 -0
- package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Info.plist +20 -0
- package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
- package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +4549 -0
- package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +3470 -0
- package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml-alloc.h +76 -0
- package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml-backend.h +354 -0
- package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml-blas.h +25 -0
- package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml-cpu.h +143 -0
- package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml-metal.h +66 -0
- package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml.h +2192 -0
- package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/gguf.h +202 -0
- package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/llama.h +1440 -0
- package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Info.plist +32 -0
- package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Modules/module.modulemap +17 -0
- package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/llama +0 -0
- package/lib/module/NativeRNLlamaCpp.js +35 -0
- package/lib/module/NativeRNLlamaCpp.js.map +1 -0
- package/lib/module/index.js +20 -0
- package/lib/module/index.js.map +1 -0
- package/lib/module/package.json +1 -0
- package/lib/typescript/package.json +1 -0
- package/lib/typescript/src/NativeRNLlamaCpp.d.ts +222 -0
- package/lib/typescript/src/NativeRNLlamaCpp.d.ts.map +1 -0
- package/lib/typescript/src/index.d.ts +5 -0
- package/lib/typescript/src/index.d.ts.map +1 -0
- package/package.json +161 -0
- package/react-native.config.js +15 -0
- package/src/NativeRNLlamaCpp.ts +282 -0
- package/src/index.tsx +54 -0
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
#include <cstdio>
|
|
2
|
+
#include <vector>
|
|
3
|
+
#include <random>
|
|
4
|
+
#include <chrono>
|
|
5
|
+
#include <cstdlib>
|
|
6
|
+
#include <cmath>
|
|
7
|
+
#include <cassert>
|
|
8
|
+
#include <cstring>
|
|
9
|
+
#include <array>
|
|
10
|
+
|
|
11
|
+
#include <ggml.h>
|
|
12
|
+
#include <ggml-cpu.h>
|
|
13
|
+
|
|
14
|
+
#if defined(_MSC_VER)
|
|
15
|
+
#pragma warning(disable: 4244 4267) // possible loss of data
|
|
16
|
+
#endif
|
|
17
|
+
|
|
18
|
+
constexpr int kVecSize = 1 << 18;
|
|
19
|
+
|
|
20
|
+
static float drawFromGaussianPdf(std::mt19937& rndm) {
|
|
21
|
+
constexpr double kScale = 1./(1. + std::mt19937::max());
|
|
22
|
+
constexpr double kTwoPiTimesScale = 6.28318530717958647692*kScale;
|
|
23
|
+
static float lastX;
|
|
24
|
+
static bool haveX = false;
|
|
25
|
+
if (haveX) { haveX = false; return lastX; }
|
|
26
|
+
auto r = sqrt(-2*log(1 - kScale*rndm()));
|
|
27
|
+
auto phi = kTwoPiTimesScale * rndm();
|
|
28
|
+
lastX = r*sin(phi);
|
|
29
|
+
haveX = true;
|
|
30
|
+
return r*cos(phi);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
static void fillRandomGaussianFloats(std::vector<float>& values, std::mt19937& rndm, float mean = 0) {
|
|
34
|
+
for (auto& v : values) v = mean + drawFromGaussianPdf(rndm);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// Copy-pasted from ggml.c
|
|
38
|
+
#define QK4_0 32
|
|
39
|
+
typedef struct {
|
|
40
|
+
float d; // delta
|
|
41
|
+
uint8_t qs[QK4_0 / 2]; // nibbles / quants
|
|
42
|
+
} block_q4_0;
|
|
43
|
+
static_assert(sizeof(block_q4_0) == sizeof(float) + QK4_0 / 2, "wrong q4_0 block size/padding");
|
|
44
|
+
|
|
45
|
+
#define QK4_1 32
|
|
46
|
+
typedef struct {
|
|
47
|
+
float d; // delta
|
|
48
|
+
float m; // min
|
|
49
|
+
uint8_t qs[QK4_1 / 2]; // nibbles / quants
|
|
50
|
+
} block_q4_1;
|
|
51
|
+
static_assert(sizeof(block_q4_1) == sizeof(float) * 2 + QK4_1 / 2, "wrong q4_1 block size/padding");
|
|
52
|
+
|
|
53
|
+
// Copy-pasted from ggml.c
|
|
54
|
+
#define QK8_0 32
|
|
55
|
+
typedef struct {
|
|
56
|
+
float d; // delta
|
|
57
|
+
int8_t qs[QK8_0]; // quants
|
|
58
|
+
} block_q8_0;
|
|
59
|
+
static_assert(sizeof(block_q8_0) == sizeof(float) + QK8_0, "wrong q8_0 block size/padding");
|
|
60
|
+
|
|
61
|
+
// "Scalar" dot product between the quantized vector x and float vector y
|
|
62
|
+
inline double dot(int n, const block_q4_0* x, const float* y) {
|
|
63
|
+
const static float kValues[16] = {-8.f, -7.f, -6.f, -5.f, -4.f, -3.f, -2.f, -1.f, 0.f, 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f};
|
|
64
|
+
constexpr uint32_t kMask1 = 0x0f0f0f0f;
|
|
65
|
+
uint32_t u1, u2;
|
|
66
|
+
auto q1 = (const uint8_t*)&u1;
|
|
67
|
+
auto q2 = (const uint8_t*)&u2;
|
|
68
|
+
double sum = 0;
|
|
69
|
+
for (int i=0; i<n; ++i) {
|
|
70
|
+
float d = x->d;
|
|
71
|
+
auto u = (const uint32_t*)x->qs;
|
|
72
|
+
float s = 0;
|
|
73
|
+
for (int k=0; k<4; ++k) {
|
|
74
|
+
u1 = u[k] & kMask1;
|
|
75
|
+
u2 = (u[k] >> 4) & kMask1;
|
|
76
|
+
s += y[0]*kValues[q1[0]] + y[1]*kValues[q2[0]] +
|
|
77
|
+
y[2]*kValues[q1[1]] + y[3]*kValues[q2[1]] +
|
|
78
|
+
y[4]*kValues[q1[2]] + y[5]*kValues[q2[2]] +
|
|
79
|
+
y[6]*kValues[q1[3]] + y[7]*kValues[q2[3]];
|
|
80
|
+
y += 8;
|
|
81
|
+
}
|
|
82
|
+
sum += s*d;
|
|
83
|
+
++x;
|
|
84
|
+
}
|
|
85
|
+
return sum;
|
|
86
|
+
}
|
|
87
|
+
// Alternative version of the above. Faster on my Mac (~45 us vs ~55 us per dot product),
|
|
88
|
+
// but about the same on X86_64 (Ryzen 7950X CPU).
|
|
89
|
+
inline double dot3(int n, const block_q4_0* x, const float* y) {
|
|
90
|
+
const static std::pair<float,float> kValues[256] = {
|
|
91
|
+
{-8.f, -8.f}, {-7.f, -8.f}, {-6.f, -8.f}, {-5.f, -8.f}, {-4.f, -8.f}, {-3.f, -8.f}, {-2.f, -8.f}, {-1.f, -8.f},
|
|
92
|
+
{ 0.f, -8.f}, { 1.f, -8.f}, { 2.f, -8.f}, { 3.f, -8.f}, { 4.f, -8.f}, { 5.f, -8.f}, { 6.f, -8.f}, { 7.f, -8.f},
|
|
93
|
+
{-8.f, -7.f}, {-7.f, -7.f}, {-6.f, -7.f}, {-5.f, -7.f}, {-4.f, -7.f}, {-3.f, -7.f}, {-2.f, -7.f}, {-1.f, -7.f},
|
|
94
|
+
{ 0.f, -7.f}, { 1.f, -7.f}, { 2.f, -7.f}, { 3.f, -7.f}, { 4.f, -7.f}, { 5.f, -7.f}, { 6.f, -7.f}, { 7.f, -7.f},
|
|
95
|
+
{-8.f, -6.f}, {-7.f, -6.f}, {-6.f, -6.f}, {-5.f, -6.f}, {-4.f, -6.f}, {-3.f, -6.f}, {-2.f, -6.f}, {-1.f, -6.f},
|
|
96
|
+
{ 0.f, -6.f}, { 1.f, -6.f}, { 2.f, -6.f}, { 3.f, -6.f}, { 4.f, -6.f}, { 5.f, -6.f}, { 6.f, -6.f}, { 7.f, -6.f},
|
|
97
|
+
{-8.f, -5.f}, {-7.f, -5.f}, {-6.f, -5.f}, {-5.f, -5.f}, {-4.f, -5.f}, {-3.f, -5.f}, {-2.f, -5.f}, {-1.f, -5.f},
|
|
98
|
+
{ 0.f, -5.f}, { 1.f, -5.f}, { 2.f, -5.f}, { 3.f, -5.f}, { 4.f, -5.f}, { 5.f, -5.f}, { 6.f, -5.f}, { 7.f, -5.f},
|
|
99
|
+
{-8.f, -4.f}, {-7.f, -4.f}, {-6.f, -4.f}, {-5.f, -4.f}, {-4.f, -4.f}, {-3.f, -4.f}, {-2.f, -4.f}, {-1.f, -4.f},
|
|
100
|
+
{ 0.f, -4.f}, { 1.f, -4.f}, { 2.f, -4.f}, { 3.f, -4.f}, { 4.f, -4.f}, { 5.f, -4.f}, { 6.f, -4.f}, { 7.f, -4.f},
|
|
101
|
+
{-8.f, -3.f}, {-7.f, -3.f}, {-6.f, -3.f}, {-5.f, -3.f}, {-4.f, -3.f}, {-3.f, -3.f}, {-2.f, -3.f}, {-1.f, -3.f},
|
|
102
|
+
{ 0.f, -3.f}, { 1.f, -3.f}, { 2.f, -3.f}, { 3.f, -3.f}, { 4.f, -3.f}, { 5.f, -3.f}, { 6.f, -3.f}, { 7.f, -3.f},
|
|
103
|
+
{-8.f, -2.f}, {-7.f, -2.f}, {-6.f, -2.f}, {-5.f, -2.f}, {-4.f, -2.f}, {-3.f, -2.f}, {-2.f, -2.f}, {-1.f, -2.f},
|
|
104
|
+
{ 0.f, -2.f}, { 1.f, -2.f}, { 2.f, -2.f}, { 3.f, -2.f}, { 4.f, -2.f}, { 5.f, -2.f}, { 6.f, -2.f}, { 7.f, -2.f},
|
|
105
|
+
{-8.f, -1.f}, {-7.f, -1.f}, {-6.f, -1.f}, {-5.f, -1.f}, {-4.f, -1.f}, {-3.f, -1.f}, {-2.f, -1.f}, {-1.f, -1.f},
|
|
106
|
+
{ 0.f, -1.f}, { 1.f, -1.f}, { 2.f, -1.f}, { 3.f, -1.f}, { 4.f, -1.f}, { 5.f, -1.f}, { 6.f, -1.f}, { 7.f, -1.f},
|
|
107
|
+
{-8.f, 0.f}, {-7.f, 0.f}, {-6.f, 0.f}, {-5.f, 0.f}, {-4.f, 0.f}, {-3.f, 0.f}, {-2.f, 0.f}, {-1.f, 0.f},
|
|
108
|
+
{ 0.f, 0.f}, { 1.f, 0.f}, { 2.f, 0.f}, { 3.f, 0.f}, { 4.f, 0.f}, { 5.f, 0.f}, { 6.f, 0.f}, { 7.f, 0.f},
|
|
109
|
+
{-8.f, 1.f}, {-7.f, 1.f}, {-6.f, 1.f}, {-5.f, 1.f}, {-4.f, 1.f}, {-3.f, 1.f}, {-2.f, 1.f}, {-1.f, 1.f},
|
|
110
|
+
{ 0.f, 1.f}, { 1.f, 1.f}, { 2.f, 1.f}, { 3.f, 1.f}, { 4.f, 1.f}, { 5.f, 1.f}, { 6.f, 1.f}, { 7.f, 1.f},
|
|
111
|
+
{-8.f, 2.f}, {-7.f, 2.f}, {-6.f, 2.f}, {-5.f, 2.f}, {-4.f, 2.f}, {-3.f, 2.f}, {-2.f, 2.f}, {-1.f, 2.f},
|
|
112
|
+
{ 0.f, 2.f}, { 1.f, 2.f}, { 2.f, 2.f}, { 3.f, 2.f}, { 4.f, 2.f}, { 5.f, 2.f}, { 6.f, 2.f}, { 7.f, 2.f},
|
|
113
|
+
{-8.f, 3.f}, {-7.f, 3.f}, {-6.f, 3.f}, {-5.f, 3.f}, {-4.f, 3.f}, {-3.f, 3.f}, {-2.f, 3.f}, {-1.f, 3.f},
|
|
114
|
+
{ 0.f, 3.f}, { 1.f, 3.f}, { 2.f, 3.f}, { 3.f, 3.f}, { 4.f, 3.f}, { 5.f, 3.f}, { 6.f, 3.f}, { 7.f, 3.f},
|
|
115
|
+
{-8.f, 4.f}, {-7.f, 4.f}, {-6.f, 4.f}, {-5.f, 4.f}, {-4.f, 4.f}, {-3.f, 4.f}, {-2.f, 4.f}, {-1.f, 4.f},
|
|
116
|
+
{ 0.f, 4.f}, { 1.f, 4.f}, { 2.f, 4.f}, { 3.f, 4.f}, { 4.f, 4.f}, { 5.f, 4.f}, { 6.f, 4.f}, { 7.f, 4.f},
|
|
117
|
+
{-8.f, 5.f}, {-7.f, 5.f}, {-6.f, 5.f}, {-5.f, 5.f}, {-4.f, 5.f}, {-3.f, 5.f}, {-2.f, 5.f}, {-1.f, 5.f},
|
|
118
|
+
{ 0.f, 5.f}, { 1.f, 5.f}, { 2.f, 5.f}, { 3.f, 5.f}, { 4.f, 5.f}, { 5.f, 5.f}, { 6.f, 5.f}, { 7.f, 5.f},
|
|
119
|
+
{-8.f, 6.f}, {-7.f, 6.f}, {-6.f, 6.f}, {-5.f, 6.f}, {-4.f, 6.f}, {-3.f, 6.f}, {-2.f, 6.f}, {-1.f, 6.f},
|
|
120
|
+
{ 0.f, 6.f}, { 1.f, 6.f}, { 2.f, 6.f}, { 3.f, 6.f}, { 4.f, 6.f}, { 5.f, 6.f}, { 6.f, 6.f}, { 7.f, 6.f},
|
|
121
|
+
{-8.f, 7.f}, {-7.f, 7.f}, {-6.f, 7.f}, {-5.f, 7.f}, {-4.f, 7.f}, {-3.f, 7.f}, {-2.f, 7.f}, {-1.f, 7.f},
|
|
122
|
+
{ 0.f, 7.f}, { 1.f, 7.f}, { 2.f, 7.f}, { 3.f, 7.f}, { 4.f, 7.f}, { 5.f, 7.f}, { 6.f, 7.f}, { 7.f, 7.f}
|
|
123
|
+
};
|
|
124
|
+
double sum = 0;
|
|
125
|
+
for (int i=0; i<n; ++i) {
|
|
126
|
+
float d = x->d;
|
|
127
|
+
auto q = x->qs;
|
|
128
|
+
float s = 0;
|
|
129
|
+
for (int k=0; k<4; ++k) {
|
|
130
|
+
s += y[0]*kValues[q[0]].first + y[1]*kValues[q[0]].second +
|
|
131
|
+
y[2]*kValues[q[1]].first + y[3]*kValues[q[1]].second +
|
|
132
|
+
y[4]*kValues[q[2]].first + y[5]*kValues[q[2]].second +
|
|
133
|
+
y[6]*kValues[q[3]].first + y[7]*kValues[q[3]].second;
|
|
134
|
+
y += 8; q += 4;
|
|
135
|
+
}
|
|
136
|
+
sum += s*d;
|
|
137
|
+
++x;
|
|
138
|
+
}
|
|
139
|
+
return sum;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
inline double dot41(int n, const block_q4_1* x, const float* y) {
|
|
143
|
+
const static float kValues[16] = {0.f, 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.f, 9.f, 10.f, 11.f, 12.f, 13.f, 14.f, 15.f};
|
|
144
|
+
constexpr uint32_t kMask1 = 0x0f0f0f0f;
|
|
145
|
+
uint32_t u1, u2;
|
|
146
|
+
auto q1 = (const uint8_t*)&u1;
|
|
147
|
+
auto q2 = (const uint8_t*)&u2;
|
|
148
|
+
double sum = 0;
|
|
149
|
+
for (int i=0; i<n; ++i) {
|
|
150
|
+
auto u = (const uint32_t*)x->qs;
|
|
151
|
+
float s = 0, s1 = 0;
|
|
152
|
+
for (int k=0; k<4; ++k) {
|
|
153
|
+
u1 = u[k] & kMask1;
|
|
154
|
+
u2 = (u[k] >> 4) & kMask1;
|
|
155
|
+
s += y[0]*kValues[q1[0]] + y[1]*kValues[q2[0]] +
|
|
156
|
+
y[2]*kValues[q1[1]] + y[3]*kValues[q2[1]] +
|
|
157
|
+
y[4]*kValues[q1[2]] + y[5]*kValues[q2[2]] +
|
|
158
|
+
y[6]*kValues[q1[3]] + y[7]*kValues[q2[3]];
|
|
159
|
+
s1 += y[0] + y[1] + y[2] + y[3] + y[4] + y[5] + y[6] + y[7];
|
|
160
|
+
y += 8;
|
|
161
|
+
}
|
|
162
|
+
sum += s*x->d + s1*x->m;
|
|
163
|
+
++x;
|
|
164
|
+
}
|
|
165
|
+
return sum;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// Copy-pasted from ggml.c
|
|
169
|
+
static void quantize_row_q8_0_reference(const float *x, block_q8_0 *y, int k) {
|
|
170
|
+
assert(k % QK8_0 == 0);
|
|
171
|
+
const int nb = k / QK8_0;
|
|
172
|
+
|
|
173
|
+
for (int i = 0; i < nb; i++) {
|
|
174
|
+
float amax = 0.0f; // absolute max
|
|
175
|
+
|
|
176
|
+
for (int l = 0; l < QK8_0; l++) {
|
|
177
|
+
const float v = x[i*QK8_0 + l];
|
|
178
|
+
amax = std::max(amax, fabsf(v));
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const float d = amax / ((1 << 7) - 1);
|
|
182
|
+
const float id = d ? 1.0f/d : 0.0f;
|
|
183
|
+
|
|
184
|
+
y[i].d = d;
|
|
185
|
+
|
|
186
|
+
for (int l = 0; l < QK8_0; ++l) {
|
|
187
|
+
const float v = x[i*QK8_0 + l]*id;
|
|
188
|
+
y[i].qs[l] = roundf(v);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// Copy-pasted from ggml.c
|
|
194
|
+
static void dot_q4_q8(const int n, float* s, const void* vx, const void* vy) {
|
|
195
|
+
const int nb = n / QK8_0;
|
|
196
|
+
const block_q4_0* x = (const block_q4_0*)vx;
|
|
197
|
+
const block_q8_0* y = (const block_q8_0*)vy;
|
|
198
|
+
float sumf = 0;
|
|
199
|
+
for (int i = 0; i < nb; i++) {
|
|
200
|
+
const float d0 = x[i].d;
|
|
201
|
+
const float d1 = y[i].d;
|
|
202
|
+
|
|
203
|
+
const uint8_t * p0 = x[i].qs;
|
|
204
|
+
const int8_t * p1 = y[i].qs;
|
|
205
|
+
|
|
206
|
+
int sumi = 0;
|
|
207
|
+
for (int j = 0; j < QK8_0/2; j++) {
|
|
208
|
+
const uint8_t v0 = p0[j];
|
|
209
|
+
|
|
210
|
+
const int i0 = (int8_t) (v0 & 0xf) - 8;
|
|
211
|
+
const int i1 = (int8_t) (v0 >> 4) - 8;
|
|
212
|
+
|
|
213
|
+
const int i2 = p1[2*j + 0];
|
|
214
|
+
const int i3 = p1[2*j + 1];
|
|
215
|
+
|
|
216
|
+
sumi += i0*i2 + i1*i3;
|
|
217
|
+
}
|
|
218
|
+
sumf += d0*d1*sumi;
|
|
219
|
+
}
|
|
220
|
+
*s = sumf;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
int main(int argc, char** argv) {
|
|
224
|
+
|
|
225
|
+
int nloop = argc > 1 ? atoi(argv[1]) : 10;
|
|
226
|
+
bool scalar = argc > 2 ? atoi(argv[2]) : false;
|
|
227
|
+
bool useQ4_1 = argc > 3 ? atoi(argv[3]) : false;
|
|
228
|
+
|
|
229
|
+
if (scalar && useQ4_1) {
|
|
230
|
+
printf("It is not possible to use Q4_1 quantization and scalar implementations\n");
|
|
231
|
+
return 1;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
std::mt19937 rndm(1234);
|
|
235
|
+
|
|
236
|
+
std::vector<float> x1(kVecSize), y1(kVecSize);
|
|
237
|
+
int n4 = useQ4_1 ? kVecSize / QK4_1 : kVecSize / QK4_0; n4 = 64*((n4 + 63)/64);
|
|
238
|
+
int n8 = kVecSize / QK8_0; n8 = 64*((n8 + 63)/64);
|
|
239
|
+
|
|
240
|
+
const auto * funcs_cpu = ggml_get_type_traits_cpu(useQ4_1 ? GGML_TYPE_Q4_1 : GGML_TYPE_Q4_0);
|
|
241
|
+
|
|
242
|
+
std::vector<block_q4_0> q40;
|
|
243
|
+
std::vector<block_q4_1> q41;
|
|
244
|
+
if (useQ4_1) q41.resize(n4);
|
|
245
|
+
else q40.resize(n4);
|
|
246
|
+
std::vector<block_q8_0> q8(n8);
|
|
247
|
+
double sumt = 0, sumt2 = 0, maxt = 0;
|
|
248
|
+
double sumqt = 0, sumqt2 = 0, maxqt = 0;
|
|
249
|
+
double sum = 0, sumq = 0, exactSum = 0;
|
|
250
|
+
for (int iloop=0; iloop<nloop; ++iloop) {
|
|
251
|
+
|
|
252
|
+
// Fill vector x with random numbers
|
|
253
|
+
fillRandomGaussianFloats(x1, rndm);
|
|
254
|
+
|
|
255
|
+
// Fill vector y with random numbers
|
|
256
|
+
fillRandomGaussianFloats(y1, rndm);
|
|
257
|
+
|
|
258
|
+
// Compute the exact dot product
|
|
259
|
+
for (int k=0; k<kVecSize; ++k) exactSum += x1[k]*y1[k];
|
|
260
|
+
|
|
261
|
+
// quantize x.
|
|
262
|
+
// Note, we do not include this in the timing as in practical application
|
|
263
|
+
// we already have the quantized model weights.
|
|
264
|
+
if (useQ4_1) {
|
|
265
|
+
funcs_cpu->from_float(x1.data(), q41.data(), kVecSize);
|
|
266
|
+
} else {
|
|
267
|
+
funcs_cpu->from_float(x1.data(), q40.data(), kVecSize);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// Now measure time the dot product needs using the "scalar" version above
|
|
271
|
+
auto t1 = std::chrono::high_resolution_clock::now();
|
|
272
|
+
if (useQ4_1) sum += dot41(kVecSize / QK4_1, q41.data(), y1.data());
|
|
273
|
+
else sum += dot(kVecSize / QK4_0, q40.data(), y1.data());
|
|
274
|
+
auto t2 = std::chrono::high_resolution_clock::now();
|
|
275
|
+
auto t = 1e-3*std::chrono::duration_cast<std::chrono::nanoseconds>(t2-t1).count();
|
|
276
|
+
sumt += t; sumt2 += t*t; maxt = std::max(maxt, t);
|
|
277
|
+
|
|
278
|
+
// And now measure the time needed to quantize y and perform the dot product with the quantized y
|
|
279
|
+
t1 = std::chrono::high_resolution_clock::now();
|
|
280
|
+
float result;
|
|
281
|
+
if (scalar) {
|
|
282
|
+
quantize_row_q8_0_reference(y1.data(), q8.data(), kVecSize);
|
|
283
|
+
dot_q4_q8(kVecSize, &result, q40.data(), q8.data());
|
|
284
|
+
}
|
|
285
|
+
else {
|
|
286
|
+
const auto * vdot = ggml_get_type_traits_cpu(funcs_cpu->vec_dot_type);
|
|
287
|
+
vdot->from_float(y1.data(), q8.data(), kVecSize);
|
|
288
|
+
if (useQ4_1) funcs_cpu->vec_dot(kVecSize, &result, 0, q41.data(), 0, q8.data(), 0, 1);
|
|
289
|
+
else funcs_cpu->vec_dot(kVecSize, &result, 0, q40.data(), 0, q8.data(), 0, 1);
|
|
290
|
+
}
|
|
291
|
+
sumq += result;
|
|
292
|
+
t2 = std::chrono::high_resolution_clock::now();
|
|
293
|
+
t = 1e-3*std::chrono::duration_cast<std::chrono::nanoseconds>(t2-t1).count();
|
|
294
|
+
sumqt += t; sumqt2 += t*t; maxqt = std::max(maxqt, t);
|
|
295
|
+
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
// Report the time (and the average of the dot products so the compiler does not come up with the idea
|
|
299
|
+
// of optimizing away the function calls after figuring that the result is not used).
|
|
300
|
+
sum /= nloop; sumq /= nloop;
|
|
301
|
+
exactSum /= nloop;
|
|
302
|
+
printf("Exact result: <dot> = %g\n",exactSum);
|
|
303
|
+
printf("<dot> = %g, %g\n",sum,sumq);
|
|
304
|
+
sumt /= nloop; sumt2 /= nloop; sumt2 -= sumt*sumt;
|
|
305
|
+
if (sumt2 > 0) sumt2 = sqrt(sumt2);
|
|
306
|
+
printf("time = %g +/- %g us. maxt = %g us\n",sumt,sumt2,maxt);
|
|
307
|
+
sumqt /= nloop; sumqt2 /= nloop; sumqt2 -= sumqt*sumqt;
|
|
308
|
+
if (sumqt2 > 0) sumqt2 = sqrt(sumqt2);
|
|
309
|
+
printf("timeq = %g +/- %g us. maxt = %g us\n",sumqt,sumqt2,maxqt);
|
|
310
|
+
return 0;
|
|
311
|
+
}
|