@novastera-oss/llamarn 0.3.1 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (347) hide show
  1. package/README.md +86 -3
  2. package/RNLlamaCpp.podspec +1 -1
  3. package/android/CMakeLists.txt +11 -3
  4. package/android/generated/jni/react/renderer/components/RNLlamaCppSpec/RNLlamaCppSpecJSI.h +49 -4
  5. package/android/src/main/cpp/include/llama.h +53 -114
  6. package/android/src/main/jniLibs/arm64-v8a/libggml-base.so +0 -0
  7. package/android/src/main/jniLibs/arm64-v8a/libggml-cpu.so +0 -0
  8. package/android/src/main/jniLibs/arm64-v8a/libggml.so +0 -0
  9. package/android/src/main/jniLibs/arm64-v8a/libllama.so +0 -0
  10. package/android/src/main/jniLibs/armeabi-v7a/libggml-base.so +0 -0
  11. package/android/src/main/jniLibs/armeabi-v7a/libggml-cpu.so +0 -0
  12. package/android/src/main/jniLibs/armeabi-v7a/libggml.so +0 -0
  13. package/android/src/main/jniLibs/armeabi-v7a/libllama.so +0 -0
  14. package/android/src/main/jniLibs/x86/libggml-base.so +0 -0
  15. package/android/src/main/jniLibs/x86/libggml-cpu.so +0 -0
  16. package/android/src/main/jniLibs/x86/libggml.so +0 -0
  17. package/android/src/main/jniLibs/x86/libllama.so +0 -0
  18. package/android/src/main/jniLibs/x86_64/libggml-base.so +0 -0
  19. package/android/src/main/jniLibs/x86_64/libggml-cpu.so +0 -0
  20. package/android/src/main/jniLibs/x86_64/libggml.so +0 -0
  21. package/android/src/main/jniLibs/x86_64/libllama.so +0 -0
  22. package/cpp/LlamaCppModel.cpp +2 -10
  23. package/cpp/PureCppImpl.cpp +71 -4
  24. package/cpp/SystemUtils.cpp +3 -7
  25. package/cpp/build-info.cpp +2 -2
  26. package/cpp/llama.cpp/CMakeLists.txt +2 -0
  27. package/cpp/llama.cpp/CODEOWNERS +1 -1
  28. package/cpp/llama.cpp/Makefile +6 -1605
  29. package/cpp/llama.cpp/README.md +5 -1
  30. package/cpp/llama.cpp/common/arg.cpp +230 -51
  31. package/cpp/llama.cpp/common/chat-parser.cpp +9 -1
  32. package/cpp/llama.cpp/common/chat.cpp +539 -8
  33. package/cpp/llama.cpp/common/chat.h +8 -1
  34. package/cpp/llama.cpp/common/common.cpp +60 -15
  35. package/cpp/llama.cpp/common/common.h +64 -15
  36. package/cpp/llama.cpp/common/speculative.cpp +135 -54
  37. package/cpp/llama.cpp/common/speculative.h +8 -1
  38. package/cpp/llama.cpp/convert_hf_to_gguf.py +1216 -109
  39. package/cpp/llama.cpp/convert_hf_to_gguf_update.py +19 -6
  40. package/cpp/llama.cpp/convert_lora_to_gguf.py +1 -1
  41. package/cpp/llama.cpp/flake.nix +0 -5
  42. package/cpp/llama.cpp/ggml/CMakeLists.txt +6 -3
  43. package/cpp/llama.cpp/ggml/cmake/ggml-config.cmake.in +71 -70
  44. package/cpp/llama.cpp/ggml/include/ggml-opt.h +25 -6
  45. package/cpp/llama.cpp/ggml/include/ggml-zdnn.h +16 -0
  46. package/cpp/llama.cpp/ggml/include/ggml.h +90 -3
  47. package/cpp/llama.cpp/ggml/src/CMakeLists.txt +13 -1
  48. package/cpp/llama.cpp/ggml/src/ggml-alloc.c +1 -0
  49. package/cpp/llama.cpp/ggml/src/ggml-backend-reg.cpp +10 -0
  50. package/cpp/llama.cpp/ggml/src/ggml-backend.cpp +113 -17
  51. package/cpp/llama.cpp/ggml/src/ggml-blas/ggml-blas.cpp +4 -4
  52. package/cpp/llama.cpp/ggml/src/ggml-cann/CMakeLists.txt +14 -0
  53. package/cpp/llama.cpp/ggml/src/ggml-cann/aclnn_ops.cpp +701 -585
  54. package/cpp/llama.cpp/ggml/src/ggml-cann/aclnn_ops.h +13 -3
  55. package/cpp/llama.cpp/ggml/src/ggml-cann/common.h +52 -0
  56. package/cpp/llama.cpp/ggml/src/ggml-cann/ggml-cann.cpp +274 -91
  57. package/cpp/llama.cpp/ggml/src/ggml-common.h +17 -0
  58. package/cpp/llama.cpp/ggml/src/ggml-cpu/CMakeLists.txt +2 -2
  59. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/arm/quants.c +132 -596
  60. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/arm/repack.cpp +14 -286
  61. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/loongarch/quants.c +90 -569
  62. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/powerpc/quants.c +162 -589
  63. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/riscv/quants.c +55 -341
  64. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/riscv/repack.cpp +3 -58
  65. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/s390/quants.c +371 -298
  66. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/wasm/quants.c +54 -314
  67. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/x86/quants.c +184 -675
  68. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/x86/repack.cpp +4679 -1657
  69. package/cpp/llama.cpp/ggml/src/ggml-cpu/arch-fallback.h +33 -2
  70. package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu-impl.h +8 -0
  71. package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +26 -1
  72. package/cpp/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp +21 -24
  73. package/cpp/llama.cpp/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +16 -7
  74. package/cpp/llama.cpp/ggml/src/ggml-cpu/llamafile/sgemm.cpp +232 -123
  75. package/cpp/llama.cpp/ggml/src/ggml-cpu/ops.cpp +428 -23
  76. package/cpp/llama.cpp/ggml/src/ggml-cpu/ops.h +4 -8
  77. package/cpp/llama.cpp/ggml/src/ggml-cpu/quants.c +35 -0
  78. package/cpp/llama.cpp/ggml/src/ggml-cpu/quants.h +8 -0
  79. package/cpp/llama.cpp/ggml/src/ggml-cpu/repack.cpp +458 -46
  80. package/cpp/llama.cpp/ggml/src/ggml-cpu/repack.h +22 -0
  81. package/cpp/llama.cpp/ggml/src/ggml-cpu/simd-mappings.h +39 -14
  82. package/cpp/llama.cpp/ggml/src/ggml-cpu/traits.cpp +2 -2
  83. package/cpp/llama.cpp/ggml/src/ggml-cpu/traits.h +1 -1
  84. package/cpp/llama.cpp/ggml/src/ggml-cpu/vec.cpp +20 -1
  85. package/cpp/llama.cpp/ggml/src/ggml-cpu/vec.h +122 -5
  86. package/cpp/llama.cpp/ggml/src/ggml-cuda/CMakeLists.txt +9 -11
  87. package/cpp/llama.cpp/ggml/src/ggml-cuda/add-id.cu +58 -0
  88. package/cpp/llama.cpp/ggml/src/ggml-cuda/add-id.cuh +3 -0
  89. package/cpp/llama.cpp/ggml/src/ggml-cuda/binbcast.cu +275 -170
  90. package/cpp/llama.cpp/ggml/src/ggml-cuda/binbcast.cuh +2 -0
  91. package/cpp/llama.cpp/ggml/src/ggml-cuda/common.cuh +103 -65
  92. package/cpp/llama.cpp/ggml/src/ggml-cuda/conv-transpose-1d.cu +1 -4
  93. package/cpp/llama.cpp/ggml/src/ggml-cuda/conv2d.cu +171 -0
  94. package/cpp/llama.cpp/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  95. package/cpp/llama.cpp/ggml/src/ggml-cuda/convert.cu +33 -7
  96. package/cpp/llama.cpp/ggml/src/ggml-cuda/convert.cuh +13 -0
  97. package/cpp/llama.cpp/ggml/src/ggml-cuda/cpy-utils.cuh +2 -10
  98. package/cpp/llama.cpp/ggml/src/ggml-cuda/cpy.cu +3 -4
  99. package/cpp/llama.cpp/ggml/src/ggml-cuda/dequantize.cuh +14 -40
  100. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-common.cuh +83 -27
  101. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-mma-f16.cuh +116 -57
  102. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-tile-f16.cu +45 -18
  103. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-tile-f32.cu +56 -29
  104. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-vec-f16.cuh +61 -39
  105. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-vec-f32.cuh +70 -49
  106. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn-wmma-f16.cu +70 -21
  107. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn.cu +162 -50
  108. package/cpp/llama.cpp/ggml/src/ggml-cuda/fattn.cuh +2 -0
  109. package/cpp/llama.cpp/ggml/src/ggml-cuda/getrows.cu +5 -4
  110. package/cpp/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +208 -97
  111. package/cpp/llama.cpp/ggml/src/ggml-cuda/im2col.cu +46 -35
  112. package/cpp/llama.cpp/ggml/src/ggml-cuda/mean.cu +56 -2
  113. package/cpp/llama.cpp/ggml/src/ggml-cuda/mma.cuh +95 -51
  114. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmf.cu +427 -0
  115. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmf.cuh +5 -0
  116. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmq.cu +204 -57
  117. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmq.cuh +252 -168
  118. package/cpp/llama.cpp/ggml/src/ggml-cuda/{mmv.cu → mmvf.cu} +53 -53
  119. package/cpp/llama.cpp/ggml/src/ggml-cuda/{mmv.cuh → mmvf.cuh} +3 -3
  120. package/cpp/llama.cpp/ggml/src/ggml-cuda/mmvq.cu +10 -5
  121. package/cpp/llama.cpp/ggml/src/ggml-cuda/norm.cu +192 -19
  122. package/cpp/llama.cpp/ggml/src/ggml-cuda/norm.cuh +5 -0
  123. package/cpp/llama.cpp/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  124. package/cpp/llama.cpp/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  125. package/cpp/llama.cpp/ggml/src/ggml-cuda/pad_reflect_1d.cu +82 -0
  126. package/cpp/llama.cpp/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  127. package/cpp/llama.cpp/ggml/src/ggml-cuda/reduce_rows.cuh +53 -0
  128. package/cpp/llama.cpp/ggml/src/ggml-cuda/roll.cu +67 -0
  129. package/cpp/llama.cpp/ggml/src/ggml-cuda/roll.cuh +5 -0
  130. package/cpp/llama.cpp/ggml/src/ggml-cuda/set-rows.cu +1 -8
  131. package/cpp/llama.cpp/ggml/src/ggml-cuda/softcap.cu +34 -0
  132. package/cpp/llama.cpp/ggml/src/ggml-cuda/softcap.cuh +5 -0
  133. package/cpp/llama.cpp/ggml/src/ggml-cuda/softmax.cu +16 -10
  134. package/cpp/llama.cpp/ggml/src/ggml-cuda/ssm-scan.cu +153 -71
  135. package/cpp/llama.cpp/ggml/src/ggml-cuda/sum.cu +6 -10
  136. package/cpp/llama.cpp/ggml/src/ggml-cuda/sumrows.cu +21 -4
  137. package/cpp/llama.cpp/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  138. package/cpp/llama.cpp/ggml/src/ggml-cuda/unary.cu +75 -0
  139. package/cpp/llama.cpp/ggml/src/ggml-cuda/unary.cuh +2 -0
  140. package/cpp/llama.cpp/ggml/src/ggml-cuda/vecdotq.cuh +110 -22
  141. package/cpp/llama.cpp/ggml/src/ggml-cuda/vendors/cuda.h +4 -0
  142. package/cpp/llama.cpp/ggml/src/ggml-cuda/vendors/hip.h +14 -25
  143. package/cpp/llama.cpp/ggml/src/ggml-cuda/vendors/musa.h +2 -1
  144. package/cpp/llama.cpp/ggml/src/ggml-hip/CMakeLists.txt +10 -2
  145. package/cpp/llama.cpp/ggml/src/ggml-impl.h +61 -0
  146. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal-impl.h +31 -20
  147. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal.m +342 -131
  148. package/cpp/llama.cpp/ggml/src/ggml-metal/ggml-metal.metal +464 -134
  149. package/cpp/llama.cpp/ggml/src/ggml-musa/CMakeLists.txt +0 -4
  150. package/cpp/llama.cpp/ggml/src/ggml-opencl/CMakeLists.txt +8 -0
  151. package/cpp/llama.cpp/ggml/src/ggml-opencl/ggml-opencl.cpp +1108 -176
  152. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/add.cl +107 -0
  153. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/add_id.cl +42 -0
  154. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/div.cl +66 -0
  155. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/flash_attn_f16.cl +343 -0
  156. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/flash_attn_f32.cl +343 -0
  157. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl +346 -0
  158. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/glu.cl +41 -0
  159. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/group_norm.cl +49 -0
  160. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul.cl +73 -0
  161. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl +132 -0
  162. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl +133 -0
  163. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl +189 -0
  164. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
  165. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/norm.cl +80 -0
  166. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +10 -2
  167. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +10 -2
  168. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/softmax_f16.cl +10 -2
  169. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/softmax_f32.cl +10 -2
  170. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/sub.cl +66 -0
  171. package/cpp/llama.cpp/ggml/src/ggml-opencl/kernels/transpose.cl +20 -0
  172. package/cpp/llama.cpp/ggml/src/ggml-opt.cpp +97 -41
  173. package/cpp/llama.cpp/ggml/src/ggml-quants.c +110 -16
  174. package/cpp/llama.cpp/ggml/src/ggml-quants.h +6 -0
  175. package/cpp/llama.cpp/ggml/src/ggml-rpc/ggml-rpc.cpp +22 -9
  176. package/cpp/llama.cpp/ggml/src/ggml-sycl/backend.hpp +1 -0
  177. package/cpp/llama.cpp/ggml/src/ggml-sycl/cpy.cpp +0 -212
  178. package/cpp/llama.cpp/ggml/src/ggml-sycl/cpy.hpp +213 -1
  179. package/cpp/llama.cpp/ggml/src/ggml-sycl/ggml-sycl.cpp +117 -238
  180. package/cpp/llama.cpp/ggml/src/ggml-sycl/quantize.hpp +133 -0
  181. package/cpp/llama.cpp/ggml/src/ggml-sycl/set_rows.cpp +94 -0
  182. package/cpp/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp +1666 -633
  183. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +41 -1
  184. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp +42 -0
  185. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +13 -4
  186. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +39 -29
  187. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp +107 -43
  188. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +2 -2
  189. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp +18 -0
  190. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +21 -0
  191. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp +32 -0
  192. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/exp.comp +20 -0
  193. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +21 -0
  194. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.comp +16 -1
  195. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +44 -8
  196. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +44 -16
  197. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +26 -1
  198. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.comp +2 -17
  199. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.comp +2 -0
  200. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.comp +37 -1
  201. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +11 -7
  202. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +109 -55
  203. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +71 -41
  204. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.comp +6 -0
  205. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +111 -0
  206. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
  207. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +49 -11
  208. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
  209. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +9 -3
  210. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/sqrt.comp +17 -0
  211. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +38 -5
  212. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp +14 -0
  213. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +55 -0
  214. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/utils.comp +25 -0
  215. package/cpp/llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +75 -20
  216. package/cpp/llama.cpp/ggml/src/ggml-webgpu/CMakeLists.txt +2 -2
  217. package/cpp/llama.cpp/ggml/src/ggml-webgpu/ggml-webgpu.cpp +807 -412
  218. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py +72 -22
  219. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +8 -8
  220. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat.tmpl.wgsl +1794 -0
  221. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl +82 -0
  222. package/cpp/llama.cpp/ggml/src/ggml-zdnn/CMakeLists.txt +36 -0
  223. package/cpp/llama.cpp/ggml/src/ggml-zdnn/ggml-zdnn-impl.h +97 -0
  224. package/cpp/llama.cpp/ggml/src/ggml-zdnn/ggml-zdnn.cpp +846 -0
  225. package/cpp/llama.cpp/ggml/src/ggml.c +204 -50
  226. package/cpp/llama.cpp/gguf-py/gguf/constants.py +187 -2
  227. package/cpp/llama.cpp/gguf-py/gguf/gguf_writer.py +11 -2
  228. package/cpp/llama.cpp/gguf-py/gguf/quants.py +53 -4
  229. package/cpp/llama.cpp/gguf-py/gguf/scripts/gguf_convert_endian.py +67 -63
  230. package/cpp/llama.cpp/gguf-py/gguf/scripts/gguf_new_metadata.py +7 -1
  231. package/cpp/llama.cpp/gguf-py/gguf/tensor_mapping.py +120 -16
  232. package/cpp/llama.cpp/gguf-py/gguf/utility.py +5 -1
  233. package/cpp/llama.cpp/gguf-py/gguf/vocab.py +284 -1
  234. package/cpp/llama.cpp/gguf-py/tests/test_quants.py +14 -5
  235. package/cpp/llama.cpp/include/llama.h +53 -114
  236. package/cpp/llama.cpp/models/templates/ByteDance-Seed-OSS.jinja +171 -0
  237. package/cpp/llama.cpp/models/templates/README.md +2 -1
  238. package/cpp/llama.cpp/models/templates/ibm-granite-granite-3.3-2B-Instruct.jinja +59 -0
  239. package/cpp/llama.cpp/models/templates/openai-gpt-oss-120b.jinja +331 -0
  240. package/cpp/llama.cpp/models/templates/unsloth-mistral-Devstral-Small-2507.jinja +105 -0
  241. package/cpp/llama.cpp/requirements/requirements-convert_hf_to_gguf.txt +3 -1
  242. package/cpp/llama.cpp/requirements/requirements-convert_hf_to_gguf_update.txt +0 -6
  243. package/cpp/llama.cpp/requirements/requirements-pydantic.txt +1 -1
  244. package/cpp/llama.cpp/src/CMakeLists.txt +2 -2
  245. package/cpp/llama.cpp/src/llama-adapter.cpp +68 -4
  246. package/cpp/llama.cpp/src/llama-adapter.h +3 -0
  247. package/cpp/llama.cpp/src/llama-arch.cpp +192 -2
  248. package/cpp/llama.cpp/src/llama-arch.h +18 -0
  249. package/cpp/llama.cpp/src/llama-batch.cpp +2 -2
  250. package/cpp/llama.cpp/src/llama-chat.cpp +47 -6
  251. package/cpp/llama.cpp/src/llama-chat.h +3 -0
  252. package/cpp/llama.cpp/src/llama-context.cpp +61 -252
  253. package/cpp/llama.cpp/src/llama-context.h +10 -15
  254. package/cpp/llama.cpp/src/llama-cparams.h +0 -1
  255. package/cpp/llama.cpp/src/llama-graph.cpp +180 -85
  256. package/cpp/llama.cpp/src/llama-graph.h +90 -51
  257. package/cpp/llama.cpp/src/llama-hparams.cpp +34 -3
  258. package/cpp/llama.cpp/src/llama-hparams.h +21 -6
  259. package/cpp/llama.cpp/src/{llama-kv-cache-unified-iswa.cpp → llama-kv-cache-iswa.cpp} +79 -56
  260. package/cpp/llama.cpp/src/{llama-kv-cache-unified-iswa.h → llama-kv-cache-iswa.h} +30 -28
  261. package/cpp/llama.cpp/src/{llama-kv-cache-unified.cpp → llama-kv-cache.cpp} +240 -632
  262. package/cpp/llama.cpp/src/{llama-kv-cache-unified.h → llama-kv-cache.h} +39 -74
  263. package/cpp/llama.cpp/src/llama-kv-cells.h +21 -21
  264. package/cpp/llama.cpp/src/llama-memory-hybrid.cpp +41 -35
  265. package/cpp/llama.cpp/src/llama-memory-hybrid.h +26 -29
  266. package/cpp/llama.cpp/src/llama-memory-recurrent.cpp +13 -9
  267. package/cpp/llama.cpp/src/llama-memory-recurrent.h +10 -14
  268. package/cpp/llama.cpp/src/llama-memory.h +13 -10
  269. package/cpp/llama.cpp/src/llama-model-loader.cpp +2 -0
  270. package/cpp/llama.cpp/src/llama-model-loader.h +3 -2
  271. package/cpp/llama.cpp/src/llama-model.cpp +1959 -419
  272. package/cpp/llama.cpp/src/llama-model.h +28 -4
  273. package/cpp/llama.cpp/src/llama-quant.cpp +40 -4
  274. package/cpp/llama.cpp/src/llama-vocab.cpp +51 -2
  275. package/cpp/llama.cpp/src/llama-vocab.h +1 -0
  276. package/cpp/llama.cpp/vendor/minja/chat-template.hpp +16 -7
  277. package/cpp/llama.cpp/vendor/minja/minja.hpp +47 -12
  278. package/cpp/rn-completion.cpp +3 -27
  279. package/ios/generated/RNLlamaCppSpec/RNLlamaCppSpec.h +30 -0
  280. package/ios/generated/RNLlamaCppSpecJSI.h +49 -4
  281. package/ios/include/chat.h +8 -1
  282. package/ios/include/common/minja/chat-template.hpp +16 -7
  283. package/ios/include/common/minja/minja.hpp +47 -12
  284. package/ios/include/common.h +64 -15
  285. package/ios/include/llama.h +53 -114
  286. package/ios/include/speculative.h +8 -1
  287. package/ios/libs/llama.xcframework/Info.plist +18 -18
  288. package/ios/libs/llama.xcframework/ios-arm64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  289. package/ios/libs/llama.xcframework/ios-arm64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +5557 -5267
  290. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/ggml-opt.h +25 -6
  291. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/ggml.h +90 -3
  292. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/Headers/llama.h +53 -114
  293. package/ios/libs/llama.xcframework/ios-arm64/llama.framework/llama +0 -0
  294. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  295. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +5520 -5238
  296. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +4241 -4014
  297. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml-opt.h +25 -6
  298. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml.h +90 -3
  299. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/llama.h +53 -114
  300. package/ios/libs/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/llama +0 -0
  301. package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  302. package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +5519 -5238
  303. package/ios/libs/llama.xcframework/macos-arm64_x86_64/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +4242 -4016
  304. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml-opt.h +25 -6
  305. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/ggml.h +90 -3
  306. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Headers/llama.h +53 -114
  307. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-opt.h +25 -6
  308. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml.h +90 -3
  309. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/llama.h +53 -114
  310. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/llama +0 -0
  311. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml-opt.h +25 -6
  312. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/ggml.h +90 -3
  313. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/Headers/llama.h +53 -114
  314. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/Current/llama +0 -0
  315. package/ios/libs/llama.xcframework/macos-arm64_x86_64/llama.framework/llama +0 -0
  316. package/ios/libs/llama.xcframework/tvos-arm64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  317. package/ios/libs/llama.xcframework/tvos-arm64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +5556 -5267
  318. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml-opt.h +25 -6
  319. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/ggml.h +90 -3
  320. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/Headers/llama.h +53 -114
  321. package/ios/libs/llama.xcframework/tvos-arm64/llama.framework/llama +0 -0
  322. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  323. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +5519 -5238
  324. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +4241 -4014
  325. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml-opt.h +25 -6
  326. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/ggml.h +90 -3
  327. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/Headers/llama.h +53 -114
  328. package/ios/libs/llama.xcframework/tvos-arm64_x86_64-simulator/llama.framework/llama +0 -0
  329. package/ios/libs/llama.xcframework/xros-arm64/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  330. package/ios/libs/llama.xcframework/xros-arm64/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +5553 -5303
  331. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml-opt.h +25 -6
  332. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/ggml.h +90 -3
  333. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/Headers/llama.h +53 -114
  334. package/ios/libs/llama.xcframework/xros-arm64/llama.framework/llama +0 -0
  335. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/DWARF/llama +0 -0
  336. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/aarch64/llama.yml +5515 -5274
  337. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/dSYMs/llama.dSYM/Contents/Resources/Relocations/x86_64/llama.yml +4238 -4044
  338. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml-opt.h +25 -6
  339. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/ggml.h +90 -3
  340. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/Headers/llama.h +53 -114
  341. package/ios/libs/llama.xcframework/xros-arm64_x86_64-simulator/llama.framework/llama +0 -0
  342. package/lib/module/NativeRNLlamaCpp.js.map +1 -1
  343. package/lib/typescript/src/NativeRNLlamaCpp.d.ts +5 -0
  344. package/lib/typescript/src/NativeRNLlamaCpp.d.ts.map +1 -1
  345. package/package.json +1 -2
  346. package/src/NativeRNLlamaCpp.ts +7 -0
  347. package/cpp/llama.cpp/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat.wgsl +0 -56
package/cpp/llama.cpp/ggml/src/ggml-cpu/arch/loongarch/quants.c CHANGED
@@ -821,24 +821,15 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi
821
821
 
822
822
  sumf = hsum_float_8(acc) + summs;
823
823
 
824
- #endif
825
- for (; ib < nb; ++ib) {
826
- int sumi0 = 0;
827
- int sumi1 = 0;
828
-
829
- for (int j = 0; j < qk/2; ++j) {
830
- const int v0 = (x[ib].qs[j] & 0x0F);
831
- const int v1 = (x[ib].qs[j] >> 4);
832
-
833
- sumi0 += (v0 * y[ib].qs[j]);
834
- sumi1 += (v1 * y[ib].qs[j + qk/2]);
835
- }
836
-
837
- int sumi = sumi0 + sumi1;
838
- sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d))*sumi + GGML_CPU_FP16_TO_FP32(x[ib].m)*GGML_CPU_FP16_TO_FP32(y[ib].s);
839
- }
840
-
841
824
  *s = sumf;
825
+ #else
826
+ UNUSED(nb);
827
+ UNUSED(x);
828
+ UNUSED(y);
829
+ UNUSED(ib);
830
+ UNUSED(sumf);
831
+ ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);
832
+ #endif
842
833
  }
843
834
 
844
835
  void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
@@ -883,30 +874,15 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
883
874
 
884
875
  sumf = hsum_float_8(acc);
885
876
 
886
- #endif
887
- for (; ib < nb; ++ib) {
888
- uint32_t qh;
889
- memcpy(&qh, x[ib].qh, sizeof(qh));
890
-
891
- int sumi0 = 0;
892
- int sumi1 = 0;
893
-
894
- for (int j = 0; j < qk/2; ++j) {
895
- const uint8_t xh_0 = ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;
896
- const uint8_t xh_1 = ((qh & (1u << (j + 16))) >> (j + 12));
897
-
898
- const int32_t x0 = (int8_t)(((x[ib].qs[j] & 0x0F) | xh_0) - 16);
899
- const int32_t x1 = (int8_t)(((x[ib].qs[j] >> 4) | xh_1) - 16);
900
-
901
- sumi0 += (x0 * y[ib].qs[j]);
902
- sumi1 += (x1 * y[ib].qs[j + qk/2]);
903
- }
904
-
905
- int sumi = sumi0 + sumi1;
906
- sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d)) * sumi;
907
- }
908
-
909
877
  *s = sumf;
878
+ #else
879
+ UNUSED(nb);
880
+ UNUSED(ib);
881
+ UNUSED(sumf);
882
+ UNUSED(x);
883
+ UNUSED(y);
884
+ ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
885
+ #endif
910
886
  }
911
887
 
912
888
  void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
@@ -954,30 +930,15 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi
954
930
 
955
931
  sumf = hsum_float_8(acc) + summs;
956
932
 
957
- #endif
958
- for (; ib < nb; ++ib) {
959
- uint32_t qh;
960
- memcpy(&qh, x[ib].qh, sizeof(qh));
961
-
962
- int sumi0 = 0;
963
- int sumi1 = 0;
964
-
965
- for (int j = 0; j < qk/2; ++j) {
966
- const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10;
967
- const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10;
968
-
969
- const int32_t x0 = (x[ib].qs[j] & 0xF) | xh_0;
970
- const int32_t x1 = (x[ib].qs[j] >> 4) | xh_1;
971
-
972
- sumi0 += (x0 * y[ib].qs[j]);
973
- sumi1 += (x1 * y[ib].qs[j + qk/2]);
974
- }
975
-
976
- int sumi = sumi0 + sumi1;
977
- sumf += (GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d))*sumi + GGML_CPU_FP16_TO_FP32(x[ib].m)*GGML_CPU_FP16_TO_FP32(y[ib].s);
978
- }
979
-
980
933
  *s = sumf;
934
+ #else
935
+ UNUSED(nb);
936
+ UNUSED(ib);
937
+ UNUSED(sumf);
938
+ UNUSED(x);
939
+ UNUSED(y);
940
+ ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc);
941
+ #endif
981
942
  }
982
943
 
983
944
  void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
@@ -1016,18 +977,15 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
1016
977
 
1017
978
  sumf = hsum_float_8(acc);
1018
979
 
1019
- #endif
1020
- for (; ib < nb; ++ib) {
1021
- int sumi = 0;
1022
-
1023
- for (int j = 0; j < qk; j++) {
1024
- sumi += x[ib].qs[j]*y[ib].qs[j];
1025
- }
1026
-
1027
- sumf += sumi*(GGML_CPU_FP16_TO_FP32(x[ib].d)*GGML_CPU_FP16_TO_FP32(y[ib].d));
1028
- }
1029
-
1030
980
  *s = sumf;
981
+ #else
982
+ UNUSED(nb);
983
+ UNUSED(ib);
984
+ UNUSED(sumf);
985
+ UNUSED(x);
986
+ UNUSED(y);
987
+ ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc);
988
+ #endif
1031
989
  }
1032
990
 
1033
991
  void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
@@ -1103,45 +1061,10 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
1103
1061
  *s = hsum_float_8(acc);
1104
1062
 
1105
1063
  #else
1106
-
1107
- float sumf = 0;
1108
-
1109
- for (int i = 0; i < nb; ++i) {
1110
-
1111
- const uint8_t * q2 = x[i].qs;
1112
- const int8_t * q8 = y[i].qs;
1113
- const uint8_t * sc = x[i].scales;
1114
-
1115
- int summs = 0;
1116
- for (int j = 0; j < 16; ++j) {
1117
- summs += y[i].bsums[j] * (sc[j] >> 4);
1118
- }
1119
-
1120
- const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d);
1121
- const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin);
1122
-
1123
- int isum = 0;
1124
- int is = 0;
1125
- int d;
1126
- for (int k = 0; k < QK_K/128; ++k) {
1127
- int shift = 0;
1128
- for (int j = 0; j < 4; ++j) {
1129
- d = sc[is++] & 0xF;
1130
- int isuml = 0;
1131
- for (int l = 0; l < 16; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);
1132
- isum += d * isuml;
1133
- d = sc[is++] & 0xF;
1134
- isuml = 0;
1135
- for (int l = 16; l < 32; ++l) isuml += q8[l] * ((q2[l] >> shift) & 3);
1136
- isum += d * isuml;
1137
- shift += 2;
1138
- q8 += 32;
1139
- }
1140
- q2 += 32;
1141
- }
1142
- sumf += dall * isum - dmin * summs;
1143
- }
1144
- *s = sumf;
1064
+ UNUSED(x);
1065
+ UNUSED(y);
1066
+ UNUSED(nb);
1067
+ ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1145
1068
  #endif
1146
1069
  }
1147
1070
 
@@ -1239,70 +1162,13 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
1239
1162
  *s = hsum_float_8(acc);
1240
1163
 
1241
1164
  #else
1242
- // scalar version
1243
- // This function is written like this so the compiler can manage to vectorize most of it
1244
- // Using -Ofast, GCC and clang manage to produce code that is within a factor of 2 or so from the
1245
- // manually vectorized version above. Every other version I tried would run at least 4 times slower.
1246
- // The ideal situation would be if we could just write the code once, and the compiler would
1247
- // automatically produce the best possible set of machine instructions, instead of us having to manually
1248
- // write vectorized versions for AVX, ARM_NEON, etc.
1249
-
1250
- int8_t aux8[QK_K];
1251
- int16_t aux16[8];
1252
- float sums [8];
1253
- int32_t aux32[8];
1254
- memset(sums, 0, 8*sizeof(float));
1255
-
1256
- uint32_t auxs[4];
1257
- const int8_t * scales = (const int8_t*)auxs;
1258
-
1259
- float sumf = 0;
1260
- for (int i = 0; i < nb; ++i) {
1261
- const uint8_t * GGML_RESTRICT q3 = x[i].qs;
1262
- const uint8_t * GGML_RESTRICT hm = x[i].hmask;
1263
- const int8_t * GGML_RESTRICT q8 = y[i].qs;
1264
- memset(aux32, 0, 8*sizeof(int32_t));
1265
- int8_t * GGML_RESTRICT a = aux8;
1266
- uint8_t m = 1;
1267
- for (int j = 0; j < QK_K; j += 128) {
1268
- for (int l = 0; l < 32; ++l) a[l] = q3[l] & 3;
1269
- for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);
1270
- a += 32; m <<= 1;
1271
- for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 2) & 3;
1272
- for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);
1273
- a += 32; m <<= 1;
1274
- for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 4) & 3;
1275
- for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);
1276
- a += 32; m <<= 1;
1277
- for (int l = 0; l < 32; ++l) a[l] = (q3[l] >> 6) & 3;
1278
- for (int l = 0; l < 32; ++l) a[l] -= (hm[l] & m ? 0 : 4);
1279
- a += 32; m <<= 1;
1280
- q3 += 32;
1281
- }
1282
- a = aux8;
1283
-
1284
- memcpy(auxs, x[i].scales, 12);
1285
- uint32_t tmp = auxs[2];
1286
- auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);
1287
- auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);
1288
- auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);
1289
- auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);
1290
- for (int j = 0; j < QK_K/16; ++j) {
1291
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
1292
- for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];
1293
- q8 += 8; a += 8;
1294
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
1295
- for (int l = 0; l < 8; ++l) aux32[l] += (scales[j] - 32) * aux16[l];
1296
- q8 += 8; a += 8;
1297
- }
1298
- const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
1299
- for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
1300
- }
1301
- for (int l = 0; l < 8; ++l) sumf += sums[l];
1302
- *s = sumf;
1303
-
1165
+ UNUSED(kmask1);
1166
+ UNUSED(kmask2);
1167
+ UNUSED(x);
1168
+ UNUSED(y);
1169
+ UNUSED(nb);
1170
+ ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1304
1171
  #endif
1305
-
1306
1172
  }
1307
1173
 
1308
1174
  void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
@@ -1391,61 +1257,14 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
1391
1257
  *s = hsum_float_8(acc) + ((v4f32)acc_m)[0];
1392
1258
 
1393
1259
  #else
1394
-
1395
- const uint8_t * scales = (const uint8_t*)&utmp[0];
1396
- const uint8_t * mins = (const uint8_t*)&utmp[2];
1397
-
1398
- int8_t aux8[QK_K];
1399
- int16_t aux16[8];
1400
- float sums [8];
1401
- int32_t aux32[8];
1402
- memset(sums, 0, 8*sizeof(float));
1403
-
1404
- float sumf = 0;
1405
- for (int i = 0; i < nb; ++i) {
1406
- const uint8_t * GGML_RESTRICT q4 = x[i].qs;
1407
- const int8_t * GGML_RESTRICT q8 = y[i].qs;
1408
- memset(aux32, 0, 8*sizeof(int32_t));
1409
- int8_t * GGML_RESTRICT a = aux8;
1410
- for (int j = 0; j < QK_K/64; ++j) {
1411
- for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);
1412
- a += 32;
1413
- for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);
1414
- a += 32; q4 += 32;
1415
- }
1416
- memcpy(utmp, x[i].scales, 12);
1417
- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
1418
- const uint32_t uaux = utmp[1] & kmask1;
1419
- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
1420
- utmp[2] = uaux;
1421
- utmp[0] &= kmask1;
1422
-
1423
- int sumi = 0;
1424
- for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];
1425
- a = aux8;
1426
- int is = 0;
1427
- for (int j = 0; j < QK_K/32; ++j) {
1428
- int32_t scale = scales[is++];
1429
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
1430
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
1431
- q8 += 8; a += 8;
1432
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
1433
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
1434
- q8 += 8; a += 8;
1435
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
1436
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
1437
- q8 += 8; a += 8;
1438
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
1439
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
1440
- q8 += 8; a += 8;
1441
- }
1442
- const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
1443
- for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
1444
- const float dmin = GGML_CPU_FP16_TO_FP32(x[i].dmin) * y[i].d;
1445
- sumf -= dmin * sumi;
1446
- }
1447
- for (int l = 0; l < 8; ++l) sumf += sums[l];
1448
- *s = sumf;
1260
+ UNUSED(x);
1261
+ UNUSED(y);
1262
+ UNUSED(nb);
1263
+ UNUSED(kmask1);
1264
+ UNUSED(kmask2);
1265
+ UNUSED(kmask3);
1266
+ UNUSED(utmp);
1267
+ ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1449
1268
  #endif
1450
1269
  }
1451
1270
 
@@ -1541,66 +1360,14 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
1541
1360
  *s = hsum_float_8(acc) + ((v4f32)acc_m)[0];
1542
1361
 
1543
1362
  #else
1544
-
1545
- const uint8_t * scales = (const uint8_t*)&utmp[0];
1546
- const uint8_t * mins = (const uint8_t*)&utmp[2];
1547
-
1548
- int8_t aux8[QK_K];
1549
- int16_t aux16[8];
1550
- float sums [8];
1551
- int32_t aux32[8];
1552
- memset(sums, 0, 8*sizeof(float));
1553
-
1554
- float sumf = 0;
1555
- for (int i = 0; i < nb; ++i) {
1556
- const uint8_t * GGML_RESTRICT q4 = x[i].qs;
1557
- const uint8_t * GGML_RESTRICT hm = x[i].qh;
1558
- const int8_t * GGML_RESTRICT q8 = y[i].qs;
1559
- memset(aux32, 0, 8*sizeof(int32_t));
1560
- int8_t * GGML_RESTRICT a = aux8;
1561
- uint8_t m = 1;
1562
- for (int j = 0; j < QK_K/64; ++j) {
1563
- for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] & 0xF);
1564
- for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);
1565
- a += 32; m <<= 1;
1566
- for (int l = 0; l < 32; ++l) a[l] = (int8_t)(q4[l] >> 4);
1567
- for (int l = 0; l < 32; ++l) a[l] += (hm[l] & m ? 16 : 0);
1568
- a += 32; m <<= 1;
1569
- q4 += 32;
1570
- }
1571
- memcpy(utmp, x[i].scales, 12);
1572
- utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4);
1573
- const uint32_t uaux = utmp[1] & kmask1;
1574
- utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4);
1575
- utmp[2] = uaux;
1576
- utmp[0] &= kmask1;
1577
-
1578
- int sumi = 0;
1579
- for (int j = 0; j < QK_K/16; ++j) sumi += y[i].bsums[j] * mins[j/2];
1580
- a = aux8;
1581
- int is = 0;
1582
- for (int j = 0; j < QK_K/32; ++j) {
1583
- int32_t scale = scales[is++];
1584
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
1585
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
1586
- q8 += 8; a += 8;
1587
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
1588
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
1589
- q8 += 8; a += 8;
1590
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
1591
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
1592
- q8 += 8; a += 8;
1593
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
1594
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
1595
- q8 += 8; a += 8;
1596
- }
1597
- const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
1598
- for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
1599
- const float dmin = GGML_CPU_FP16_TO_FP32(x[i].dmin) * y[i].d;
1600
- sumf -= dmin * sumi;
1601
- }
1602
- for (int l = 0; l < 8; ++l) sumf += sums[l];
1603
- *s = sumf;
1363
+ UNUSED(x);
1364
+ UNUSED(y);
1365
+ UNUSED(nb);
1366
+ UNUSED(kmask1);
1367
+ UNUSED(kmask2);
1368
+ UNUSED(kmask3);
1369
+ UNUSED(utmp);
1370
+ ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1604
1371
  #endif
1605
1372
  }
1606
1373
 
@@ -1678,47 +1445,10 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
1678
1445
  *s = hsum_float_8(acc);
1679
1446
 
1680
1447
  #else
1681
-
1682
- int8_t aux8[QK_K];
1683
- int16_t aux16[8];
1684
- float sums [8];
1685
- int32_t aux32[8];
1686
- memset(sums, 0, 8*sizeof(float));
1687
-
1688
- float sumf = 0;
1689
- for (int i = 0; i < nb; ++i) {
1690
- const uint8_t * GGML_RESTRICT q4 = x[i].ql;
1691
- const uint8_t * GGML_RESTRICT qh = x[i].qh;
1692
- const int8_t * GGML_RESTRICT q8 = y[i].qs;
1693
- memset(aux32, 0, 8*sizeof(int32_t));
1694
- int8_t * GGML_RESTRICT a = aux8;
1695
- for (int j = 0; j < QK_K; j += 128) {
1696
- for (int l = 0; l < 32; ++l) {
1697
- a[l + 0] = (int8_t)((q4[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;
1698
- a[l + 32] = (int8_t)((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;
1699
- a[l + 64] = (int8_t)((q4[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;
1700
- a[l + 96] = (int8_t)((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;
1701
- }
1702
- a += 128;
1703
- q4 += 64;
1704
- qh += 32;
1705
- }
1706
- a = aux8;
1707
- int is = 0;
1708
- for (int j = 0; j < QK_K/16; ++j) {
1709
- int scale = x[i].scales[is++];
1710
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
1711
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
1712
- q8 += 8; a += 8;
1713
- for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l];
1714
- for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l];
1715
- q8 += 8; a += 8;
1716
- }
1717
- const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
1718
- for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l];
1719
- }
1720
- for (int l = 0; l < 8; ++l) sumf += sums[l];
1721
- *s = sumf;
1448
+ UNUSED(x);
1449
+ UNUSED(y);
1450
+ UNUSED(nb);
1451
+ ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1722
1452
  #endif
1723
1453
  }
1724
1454
 
@@ -1815,34 +1545,10 @@ void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const
1815
1545
  *s = 0.125f * hsum_float_8(accumf);
1816
1546
 
1817
1547
  #else
1818
-
1819
- uint32_t aux32[2];
1820
- const uint8_t * aux8 = (const uint8_t *)aux32;
1821
-
1822
- float sumf = 0.f;
1823
- for (int i = 0; i < nb; ++i) {
1824
- const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
1825
- const uint16_t * GGML_RESTRICT q2 = x[i].qs;
1826
- const int8_t * GGML_RESTRICT q8 = y[i].qs;
1827
- int32_t bsum = 0;
1828
- for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {
1829
- memcpy(aux32, q2, 2*sizeof(uint32_t));
1830
- q2 += 4;
1831
- const uint32_t ls = 2*(aux32[1] >> 28) + 1;
1832
- int32_t sumi = 0;
1833
- for (int l = 0; l < 4; ++l) {
1834
- const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]);
1835
- const uint8_t signs = ksigns_iq2xs[(aux32[1] >> 7*l) & 127];
1836
- for (int j = 0; j < 8; ++j) {
1837
- sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
1838
- }
1839
- q8 += 8;
1840
- }
1841
- bsum += sumi * ls;
1842
- }
1843
- sumf += d * bsum;
1844
- }
1845
- *s = 0.125f * sumf;
1548
+ UNUSED(x);
1549
+ UNUSED(y);
1550
+ UNUSED(nb);
1551
+ ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
1846
1552
  #endif
1847
1553
  }
1848
1554
 
@@ -1978,42 +1684,10 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v
1978
1684
  *s = 0.125f * hsum_float_8(accumf);
1979
1685
 
1980
1686
  #else
1981
-
1982
- float sumf = 0.f;
1983
- for (int i = 0; i < nb; ++i) {
1984
- const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
1985
- const uint16_t * GGML_RESTRICT q2 = x[i].qs;
1986
- const uint8_t * GGML_RESTRICT sc = x[i].scales;
1987
- const int8_t * GGML_RESTRICT q8 = y[i].qs;
1988
- int32_t bsum = 0;
1989
- for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {
1990
- const uint16_t ls1 = 2*(sc[ib32] & 0xf) + 1;
1991
- const uint16_t ls2 = 2*(sc[ib32] >> 4) + 1;
1992
- int32_t sumi = 0;
1993
- for (int l = 0; l < 2; ++l) {
1994
- const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));
1995
- const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];
1996
- for (int j = 0; j < 8; ++j) {
1997
- sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
1998
- }
1999
- q8 += 8;
2000
- }
2001
- bsum += sumi * ls1;
2002
- sumi = 0;
2003
- for (int l = 2; l < 4; ++l) {
2004
- const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[l] & 511));
2005
- const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];
2006
- for (int j = 0; j < 8; ++j) {
2007
- sumi += grid[j] * q8[j] * (signs & kmask_iq2xs[j] ? -1 : 1);
2008
- }
2009
- q8 += 8;
2010
- }
2011
- bsum += sumi * ls2;
2012
- q2 += 4;
2013
- }
2014
- sumf += d * bsum;
2015
- }
2016
- *s = 0.125f * sumf;
1687
+ UNUSED(x);
1688
+ UNUSED(y);
1689
+ UNUSED(nb);
1690
+ ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
2017
1691
  #endif
2018
1692
  }
2019
1693
 
@@ -2105,47 +1779,11 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo
2105
1779
  *s = 0.125f * hsum_float_8(accumf);
2106
1780
 
2107
1781
  #else
2108
-
2109
- float sumf = 0;
2110
- for (int i = 0; i < nb; i++) {
2111
-
2112
- const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
2113
- const int8_t * q8 = y[i].qs;
2114
- const uint8_t * qs = x[i].qs;
2115
- const uint8_t * qh = x[i].qh;
2116
- const uint8_t * signs = qs + QK_K/8;
2117
-
2118
- int bsum = 0;
2119
- for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {
2120
- int ls1 = 1 + 2*(x[i].scales[ib32] & 0xf);
2121
- int ls2 = 1 + 2*(x[i].scales[ib32] >> 4);
2122
- int sumi1 = 0, sumi2 = 0;
2123
- for (int l = 0; l < 2; ++l) {
2124
- const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300)));
2125
- for (int j = 0; j < 8; ++j) {
2126
- sumi1 += q8[j] * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1 : 1);
2127
- }
2128
- q8 += 8;
2129
- }
2130
- for (int l = 2; l < 4; ++l) {
2131
- const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300)));
2132
- for (int j = 0; j < 8; ++j) {
2133
- sumi2 += q8[j] * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1 : 1);
2134
- }
2135
- q8 += 8;
2136
- }
2137
- bsum += ls1 * sumi1 + ls2 * sumi2;
2138
- qs += 4;
2139
- signs += 4;
2140
- }
2141
-
2142
- sumf += d * bsum;
2143
- }
2144
-
2145
- *s = 0.125f * sumf;
2146
-
1782
+ UNUSED(x);
1783
+ UNUSED(y);
1784
+ UNUSED(nb);
1785
+ ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
2147
1786
  #endif
2148
-
2149
1787
  }
2150
1788
 
2151
1789
  void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
@@ -2209,36 +1847,10 @@ void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const
2209
1847
  *s = 0.25f * hsum_float_8(accumf);
2210
1848
 
2211
1849
  #else
2212
-
2213
- uint32_t aux32;
2214
-
2215
- float sumf = 0.f;
2216
- for (int i = 0; i < nb; ++i) {
2217
- const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
2218
- const uint8_t * GGML_RESTRICT q3 = x[i].qs;
2219
- const uint8_t * GGML_RESTRICT gas = x[i].qs + QK_K/4;
2220
- const int8_t * GGML_RESTRICT q8 = y[i].qs;
2221
- int32_t bsum = 0;
2222
- for (int ib32 = 0; ib32 < QK_K/32; ++ib32) {
2223
- memcpy(&aux32, gas, sizeof(uint32_t)); gas += sizeof(uint32_t);
2224
- const uint32_t ls = 2*(aux32 >> 28) + 1;
2225
- int32_t sumi = 0;
2226
- for (int l = 0; l < 4; ++l) {
2227
- const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*l+0]);
2228
- const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*l+1]);
2229
- const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*l) & 127];
2230
- for (int j = 0; j < 4; ++j) {
2231
- sumi += grid1[j] * q8[j+0] * (signs & kmask_iq2xs[j+0] ? -1 : 1);
2232
- sumi += grid2[j] * q8[j+4] * (signs & kmask_iq2xs[j+4] ? -1 : 1);
2233
- }
2234
- q8 += 8;
2235
- }
2236
- q3 += 8;
2237
- bsum += sumi * ls;
2238
- }
2239
- sumf += d * bsum;
2240
- }
2241
- *s = 0.25f * sumf;
1850
+ UNUSED(x);
1851
+ UNUSED(y);
1852
+ UNUSED(nb);
1853
+ ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
2242
1854
  #endif
2243
1855
  }
2244
1856
 
@@ -2338,48 +1950,10 @@ void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo
2338
1950
  *s = hsum_float_8(accumf);
2339
1951
 
2340
1952
  #else
2341
-
2342
- float sumf = 0.f;
2343
- for (int i = 0; i < nb; ++i) {
2344
- const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
2345
- const uint8_t * GGML_RESTRICT qs = x[i].qs;
2346
- const uint8_t * GGML_RESTRICT qh = x[i].qh;
2347
- const uint8_t * GGML_RESTRICT signs = x[i].signs;
2348
- const int8_t * GGML_RESTRICT q8 = y[i].qs;
2349
- int32_t bsum = 0;
2350
- for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) {
2351
- const uint32_t ls1 = 2*(x[i].scales[ib32/2] & 0xf) + 1;
2352
- const uint32_t ls2 = 2*(x[i].scales[ib32/2] >> 4) + 1;
2353
- int32_t sumi = 0;
2354
- for (int l = 0; l < 4; ++l) {
2355
- const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[ib32+0] << (8-2*l)) & 256)));
2356
- const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[ib32+0] << (7-2*l)) & 256)));
2357
- for (int j = 0; j < 4; ++j) {
2358
- sumi += grid1[j] * q8[j+0] * (signs[l] & kmask_iq2xs[j+0] ? -1 : 1);
2359
- sumi += grid2[j] * q8[j+4] * (signs[l] & kmask_iq2xs[j+4] ? -1 : 1);
2360
- }
2361
- q8 += 8;
2362
- }
2363
- qs += 8;
2364
- signs += 4;
2365
- bsum += sumi * ls1;
2366
- sumi = 0;
2367
- for (int l = 0; l < 4; ++l) {
2368
- const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[ib32+1] << (8-2*l)) & 256)));
2369
- const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[ib32+1] << (7-2*l)) & 256)));
2370
- for (int j = 0; j < 4; ++j) {
2371
- sumi += grid1[j] * q8[j+0] * (signs[l] & kmask_iq2xs[j+0] ? -1 : 1);
2372
- sumi += grid2[j] * q8[j+4] * (signs[l] & kmask_iq2xs[j+4] ? -1 : 1);
2373
- }
2374
- q8 += 8;
2375
- }
2376
- qs += 8;
2377
- signs += 4;
2378
- bsum += sumi * ls2;
2379
- }
2380
- sumf += d * bsum;
2381
- }
2382
- *s = sumf;
1953
+ UNUSED(x);
1954
+ UNUSED(y);
1955
+ UNUSED(nb);
1956
+ ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
2383
1957
  #endif
2384
1958
  }
2385
1959
 
@@ -2460,36 +2034,10 @@ void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo
2460
2034
  *s = hsum_float_8(accum) + IQ1S_DELTA * accum1;
2461
2035
 
2462
2036
  #else
2463
-
2464
- float sumf = 0;
2465
- for (int i = 0; i < nb; i++) {
2466
-
2467
- const int8_t * q8 = y[i].qs;
2468
- const uint8_t * qs = x[i].qs;
2469
- const uint16_t * qh = x[i].qh;
2470
-
2471
- int sumi = 0, sumi1 = 0;
2472
- for (int ib = 0; ib < QK_K/32; ++ib) {
2473
- const int ls = 2*((qh[ib] >> 12) & 7) + 1;
2474
- const int delta = qh[ib] & 0x8000 ? -1 : 1;
2475
- int lsum = 0;
2476
- for (int l = 0; l < 4; ++l) {
2477
- const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((qh[ib] >> 3*l) & 7) << 8)));
2478
- for (int j = 0; j < 8; ++j) {
2479
- lsum += q8[j] * grid[j];
2480
- }
2481
- q8 += 8;
2482
- }
2483
- sumi += ls * lsum;
2484
- sumi1 += ls * delta * (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]);
2485
- qs += 4;
2486
- }
2487
-
2488
- sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d * (sumi + IQ1S_DELTA * sumi1);
2489
- }
2490
-
2491
- *s = sumf;
2492
-
2037
+ UNUSED(x);
2038
+ UNUSED(y);
2039
+ UNUSED(nb);
2040
+ ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
2493
2041
  #endif
2494
2042
  }
2495
2043
 
@@ -2603,37 +2151,10 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v
2603
2151
  *s = hsum_float_8(accum);
2604
2152
 
2605
2153
  #else
2606
- float sumf = 0;
2607
- for (int ibl = 0; ibl < nb; ++ibl) {
2608
- const float d4d8 = GGML_CPU_FP16_TO_FP32(x[ibl].d) * y[ibl].d;
2609
- uint16_t h = x[ibl].scales_h;
2610
- const uint8_t * qs = x[ibl].qs;
2611
- const int8_t * q8 = y[ibl].qs;
2612
- for (int ib = 0; ib < QK_K/32; ib += 2) {
2613
- const uint8_t ls1 = (x[ibl].scales_l[ib/2] & 0xf) | ((h << 4) & 0x30);
2614
- const uint8_t ls2 = (x[ibl].scales_l[ib/2] >> 4) | ((h << 2) & 0x30);
2615
- h >>= 4;
2616
- const float d1 = d4d8*(ls1 - 32);
2617
- const float d2 = d4d8*(ls2 - 32);
2618
- int sumi1 = 0, sumi2 = 0;
2619
- for (int j = 0; j < 16; ++j) {
2620
- sumi1 += q8[j+ 0] * kvalues_iq4nl[qs[j] & 0xf];
2621
- sumi2 += q8[j+16] * kvalues_iq4nl[qs[j] >> 4];
2622
- }
2623
- sumf += d1 * (sumi1 + sumi2);
2624
- qs += 16;
2625
- q8 += 32;
2626
- sumi1 = sumi2 = 0;
2627
- for (int j = 0; j < 16; ++j) {
2628
- sumi1 += q8[j+ 0] * kvalues_iq4nl[qs[j] & 0xf];
2629
- sumi2 += q8[j+16] * kvalues_iq4nl[qs[j] >> 4];
2630
- }
2631
- sumf += d2 * (sumi1 + sumi2);
2632
- qs += 16;
2633
- q8 += 32;
2634
- }
2635
- }
2636
- *s = sumf;
2154
+ UNUSED(x);
2155
+ UNUSED(y);
2156
+ UNUSED(nb);
2157
+ ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc);
2637
2158
  #endif
2638
2159
  }
2639
2160