whispercpp 1.3.1 → 1.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (797) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -3
  3. data/README.md +92 -31
  4. data/Rakefile +26 -7
  5. data/ext/.gitignore +5 -7
  6. data/ext/dependencies.rb +61 -0
  7. data/ext/extconf.rb +21 -198
  8. data/ext/options.rb +221 -0
  9. data/ext/ruby_whisper.c +159 -0
  10. data/ext/ruby_whisper.h +17 -2
  11. data/ext/ruby_whisper_context.c +641 -0
  12. data/ext/ruby_whisper_error.c +52 -0
  13. data/ext/ruby_whisper_model.c +232 -0
  14. data/ext/ruby_whisper_params.c +1301 -0
  15. data/ext/ruby_whisper_segment.c +143 -0
  16. data/ext/ruby_whisper_transcribe.cpp +87 -0
  17. data/ext/ruby_whisper_vad_params.c +288 -0
  18. data/ext/sources/.dockerignore +3 -0
  19. data/ext/sources/.github/workflows/bindings-ruby.yml +21 -0
  20. data/ext/sources/CMakeGraphVizOptions.cmake +8 -0
  21. data/ext/sources/CMakeLists.txt +251 -0
  22. data/ext/sources/bindings/javascript/CMakeLists.txt +41 -0
  23. data/ext/sources/bindings/javascript/emscripten.cpp +93 -0
  24. data/ext/sources/bindings/javascript/libwhisper.worker.js +1 -0
  25. data/ext/sources/bindings/javascript/package-tmpl.json +26 -0
  26. data/ext/sources/bindings/javascript/package.json +26 -0
  27. data/ext/sources/bindings/javascript/whisper.js +19 -0
  28. data/ext/sources/build-xcframework.sh +547 -0
  29. data/ext/sources/ci/run.sh +336 -0
  30. data/ext/sources/close-issue.yml +28 -0
  31. data/ext/sources/cmake/DefaultTargetOptions.cmake +16 -0
  32. data/ext/sources/cmake/FindFFmpeg.cmake +163 -0
  33. data/ext/sources/cmake/build-info.cmake +60 -0
  34. data/ext/sources/cmake/git-vars.cmake +22 -0
  35. data/ext/sources/cmake/whisper-config.cmake.in +65 -0
  36. data/ext/sources/cmake/whisper.pc.in +10 -0
  37. data/ext/sources/examples/CMakeLists.txt +124 -0
  38. data/ext/sources/examples/addon.node/CMakeLists.txt +31 -0
  39. data/ext/sources/examples/addon.node/__test__/whisper.spec.js +37 -0
  40. data/ext/sources/examples/addon.node/addon.cpp +438 -0
  41. data/ext/sources/examples/addon.node/index.js +54 -0
  42. data/ext/sources/examples/addon.node/package.json +16 -0
  43. data/ext/sources/examples/bench/CMakeLists.txt +8 -0
  44. data/ext/sources/examples/bench/bench.cpp +175 -0
  45. data/ext/sources/examples/bench.wasm/CMakeLists.txt +49 -0
  46. data/ext/sources/examples/bench.wasm/emscripten.cpp +87 -0
  47. data/ext/sources/examples/bench.wasm/index-tmpl.html +284 -0
  48. data/ext/sources/examples/cli/CMakeLists.txt +8 -0
  49. data/ext/sources/examples/cli/cli.cpp +1294 -0
  50. data/ext/sources/examples/coi-serviceworker.js +146 -0
  51. data/ext/sources/examples/command/CMakeLists.txt +10 -0
  52. data/ext/sources/examples/command/command.cpp +776 -0
  53. data/ext/sources/examples/command/commands.txt +9 -0
  54. data/ext/sources/examples/command.wasm/CMakeLists.txt +50 -0
  55. data/ext/sources/examples/command.wasm/emscripten.cpp +327 -0
  56. data/ext/sources/examples/command.wasm/index-tmpl.html +414 -0
  57. data/ext/sources/examples/common-ggml.cpp +238 -0
  58. data/ext/sources/examples/common-ggml.h +18 -0
  59. data/ext/sources/examples/common-sdl.cpp +227 -0
  60. data/ext/sources/examples/common-sdl.h +49 -0
  61. data/ext/sources/examples/common-whisper.cpp +168 -0
  62. data/ext/sources/examples/common-whisper.h +24 -0
  63. data/ext/sources/examples/common.cpp +675 -0
  64. data/ext/sources/examples/common.h +322 -0
  65. data/ext/sources/examples/deprecation-warning/CMakeLists.txt +6 -0
  66. data/ext/sources/examples/deprecation-warning/deprecation-warning.cpp +38 -0
  67. data/ext/sources/examples/ffmpeg-transcode.cpp +368 -0
  68. data/ext/sources/examples/generate-karaoke.sh +57 -0
  69. data/ext/sources/examples/grammar-parser.cpp +423 -0
  70. data/ext/sources/examples/grammar-parser.h +29 -0
  71. data/ext/sources/examples/helpers.js +191 -0
  72. data/ext/sources/examples/json.hpp +24596 -0
  73. data/ext/sources/examples/livestream.sh +112 -0
  74. data/ext/sources/examples/lsp/CMakeLists.txt +9 -0
  75. data/ext/sources/examples/lsp/lsp.cpp +467 -0
  76. data/ext/sources/examples/lsp/whisper.vim +362 -0
  77. data/ext/sources/examples/miniaudio.h +93468 -0
  78. data/ext/sources/examples/python/test_whisper_processor.py +7 -0
  79. data/ext/sources/examples/python/whisper_processor.py +54 -0
  80. data/ext/sources/examples/quantize/CMakeLists.txt +6 -0
  81. data/ext/sources/examples/quantize/quantize.cpp +223 -0
  82. data/ext/sources/examples/server/CMakeLists.txt +12 -0
  83. data/ext/sources/examples/server/bench.js +29 -0
  84. data/ext/sources/examples/server/httplib.h +10497 -0
  85. data/ext/sources/examples/server/server.cpp +1091 -0
  86. data/ext/sources/examples/server.py +115 -0
  87. data/ext/sources/examples/stb_vorbis.c +5584 -0
  88. data/ext/sources/examples/stream/CMakeLists.txt +10 -0
  89. data/ext/sources/examples/stream/stream.cpp +429 -0
  90. data/ext/sources/examples/stream.wasm/CMakeLists.txt +49 -0
  91. data/ext/sources/examples/stream.wasm/emscripten.cpp +216 -0
  92. data/ext/sources/examples/stream.wasm/index-tmpl.html +414 -0
  93. data/ext/sources/examples/sycl/CMakeLists.txt +9 -0
  94. data/ext/sources/examples/sycl/build.sh +22 -0
  95. data/ext/sources/examples/sycl/ls-sycl-device.cpp +11 -0
  96. data/ext/sources/examples/sycl/run-whisper.sh +17 -0
  97. data/ext/sources/examples/talk-llama/CMakeLists.txt +40 -0
  98. data/ext/sources/examples/talk-llama/eleven-labs.py +80 -0
  99. data/ext/sources/examples/talk-llama/llama-adapter.cpp +388 -0
  100. data/ext/sources/examples/talk-llama/llama-adapter.h +76 -0
  101. data/ext/sources/examples/talk-llama/llama-arch.cpp +1746 -0
  102. data/ext/sources/examples/talk-llama/llama-arch.h +437 -0
  103. data/ext/sources/examples/talk-llama/llama-batch.cpp +374 -0
  104. data/ext/sources/examples/talk-llama/llama-batch.h +89 -0
  105. data/ext/sources/examples/talk-llama/llama-chat.cpp +663 -0
  106. data/ext/sources/examples/talk-llama/llama-chat.h +58 -0
  107. data/ext/sources/examples/talk-llama/llama-context.cpp +2676 -0
  108. data/ext/sources/examples/talk-llama/llama-context.h +276 -0
  109. data/ext/sources/examples/talk-llama/llama-cparams.cpp +5 -0
  110. data/ext/sources/examples/talk-llama/llama-cparams.h +41 -0
  111. data/ext/sources/examples/talk-llama/llama-grammar.cpp +1229 -0
  112. data/ext/sources/examples/talk-llama/llama-grammar.h +173 -0
  113. data/ext/sources/examples/talk-llama/llama-graph.cpp +1618 -0
  114. data/ext/sources/examples/talk-llama/llama-graph.h +640 -0
  115. data/ext/sources/examples/talk-llama/llama-hparams.cpp +95 -0
  116. data/ext/sources/examples/talk-llama/llama-hparams.h +190 -0
  117. data/ext/sources/examples/talk-llama/llama-impl.cpp +167 -0
  118. data/ext/sources/examples/talk-llama/llama-impl.h +61 -0
  119. data/ext/sources/examples/talk-llama/llama-io.cpp +15 -0
  120. data/ext/sources/examples/talk-llama/llama-io.h +35 -0
  121. data/ext/sources/examples/talk-llama/llama-kv-cache.cpp +2739 -0
  122. data/ext/sources/examples/talk-llama/llama-kv-cache.h +502 -0
  123. data/ext/sources/examples/talk-llama/llama-kv-cells.h +379 -0
  124. data/ext/sources/examples/talk-llama/llama-memory.cpp +1 -0
  125. data/ext/sources/examples/talk-llama/llama-memory.h +32 -0
  126. data/ext/sources/examples/talk-llama/llama-mmap.cpp +600 -0
  127. data/ext/sources/examples/talk-llama/llama-mmap.h +68 -0
  128. data/ext/sources/examples/talk-llama/llama-model-loader.cpp +1138 -0
  129. data/ext/sources/examples/talk-llama/llama-model-loader.h +169 -0
  130. data/ext/sources/examples/talk-llama/llama-model-saver.cpp +281 -0
  131. data/ext/sources/examples/talk-llama/llama-model-saver.h +37 -0
  132. data/ext/sources/examples/talk-llama/llama-model.cpp +13814 -0
  133. data/ext/sources/examples/talk-llama/llama-model.h +425 -0
  134. data/ext/sources/examples/talk-llama/llama-quant.cpp +966 -0
  135. data/ext/sources/examples/talk-llama/llama-quant.h +1 -0
  136. data/ext/sources/examples/talk-llama/llama-sampling.cpp +2575 -0
  137. data/ext/sources/examples/talk-llama/llama-sampling.h +32 -0
  138. data/ext/sources/examples/talk-llama/llama-vocab.cpp +3340 -0
  139. data/ext/sources/examples/talk-llama/llama-vocab.h +131 -0
  140. data/ext/sources/examples/talk-llama/llama.cpp +354 -0
  141. data/ext/sources/examples/talk-llama/llama.h +1377 -0
  142. data/ext/sources/examples/talk-llama/prompts/talk-alpaca.txt +23 -0
  143. data/ext/sources/examples/talk-llama/speak +40 -0
  144. data/ext/sources/examples/talk-llama/speak.bat +1 -0
  145. data/ext/sources/examples/talk-llama/speak.ps1 +14 -0
  146. data/ext/sources/examples/talk-llama/talk-llama.cpp +808 -0
  147. data/ext/sources/examples/talk-llama/unicode-data.cpp +7034 -0
  148. data/ext/sources/examples/talk-llama/unicode-data.h +20 -0
  149. data/ext/sources/examples/talk-llama/unicode.cpp +849 -0
  150. data/ext/sources/examples/talk-llama/unicode.h +66 -0
  151. data/ext/sources/examples/vad-speech-segments/CMakeLists.txt +8 -0
  152. data/ext/sources/examples/vad-speech-segments/speech.cpp +143 -0
  153. data/ext/sources/examples/wchess/CMakeLists.txt +10 -0
  154. data/ext/sources/examples/wchess/libwchess/CMakeLists.txt +19 -0
  155. data/ext/sources/examples/wchess/libwchess/Chessboard.cpp +803 -0
  156. data/ext/sources/examples/wchess/libwchess/Chessboard.h +33 -0
  157. data/ext/sources/examples/wchess/libwchess/WChess.cpp +193 -0
  158. data/ext/sources/examples/wchess/libwchess/WChess.h +63 -0
  159. data/ext/sources/examples/wchess/libwchess/test-chessboard.cpp +117 -0
  160. data/ext/sources/examples/wchess/wchess.cmd/CMakeLists.txt +8 -0
  161. data/ext/sources/examples/wchess/wchess.cmd/wchess.cmd.cpp +249 -0
  162. data/ext/sources/examples/whisper.wasm/CMakeLists.txt +50 -0
  163. data/ext/sources/examples/whisper.wasm/emscripten.cpp +118 -0
  164. data/ext/sources/examples/whisper.wasm/index-tmpl.html +658 -0
  165. data/ext/sources/ggml/CMakeLists.txt +390 -0
  166. data/ext/sources/ggml/cmake/BuildTypes.cmake +54 -0
  167. data/ext/sources/ggml/cmake/GitVars.cmake +22 -0
  168. data/ext/sources/ggml/cmake/common.cmake +26 -0
  169. data/ext/sources/ggml/cmake/ggml-config.cmake.in +152 -0
  170. data/ext/{ggml → sources/ggml}/include/ggml-alloc.h +1 -1
  171. data/ext/{ggml → sources/ggml}/include/ggml-backend.h +9 -7
  172. data/ext/{ggml → sources/ggml}/include/ggml-cpp.h +2 -1
  173. data/ext/{ggml → sources/ggml}/include/ggml-cpu.h +9 -1
  174. data/ext/{ggml → sources/ggml}/include/ggml-metal.h +1 -1
  175. data/ext/{ggml → sources/ggml}/include/ggml-opt.h +49 -28
  176. data/ext/{ggml → sources/ggml}/include/ggml-rpc.h +6 -1
  177. data/ext/{ggml → sources/ggml}/include/ggml-vulkan.h +0 -2
  178. data/ext/{ggml → sources/ggml}/include/ggml.h +182 -265
  179. data/ext/sources/ggml/include/gguf.h +202 -0
  180. data/ext/sources/ggml/src/CMakeLists.txt +346 -0
  181. data/ext/{ggml → sources/ggml}/src/ggml-alloc.c +34 -29
  182. data/ext/sources/ggml/src/ggml-amx/CMakeLists.txt +107 -0
  183. data/ext/{ggml → sources/ggml}/src/ggml-backend-impl.h +1 -2
  184. data/ext/{ggml → sources/ggml}/src/ggml-backend-reg.cpp +87 -53
  185. data/ext/{ggml → sources/ggml}/src/ggml-backend.cpp +26 -14
  186. data/ext/sources/ggml/src/ggml-blas/CMakeLists.txt +87 -0
  187. data/ext/sources/ggml/src/ggml-cann/CMakeLists.txt +74 -0
  188. data/ext/sources/ggml/src/ggml-cann/Doxyfile +2579 -0
  189. data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.cpp +10 -4
  190. data/ext/{ggml → sources/ggml}/src/ggml-cann/acl_tensor.h +5 -5
  191. data/ext/{ggml → sources/ggml}/src/ggml-cann/aclnn_ops.cpp +1272 -1506
  192. data/ext/sources/ggml/src/ggml-cann/aclnn_ops.h +1125 -0
  193. data/ext/{ggml → sources/ggml}/src/ggml-cann/common.h +135 -1
  194. data/ext/{ggml → sources/ggml}/src/ggml-cann/ggml-cann.cpp +564 -146
  195. data/ext/sources/ggml/src/ggml-cann/kernels/CMakeLists.txt +30 -0
  196. data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/dup.cpp +3 -5
  197. data/ext/{ggml → sources/ggml}/src/ggml-common.h +12 -8
  198. data/ext/sources/ggml/src/ggml-cpu/CMakeLists.txt +504 -0
  199. data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.cpp +2 -1
  200. data/ext/sources/ggml/src/ggml-cpu/binary-ops.cpp +158 -0
  201. data/ext/sources/ggml/src/ggml-cpu/binary-ops.h +16 -0
  202. data/ext/sources/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  203. data/ext/sources/ggml/src/ggml-cpu/common.h +72 -0
  204. data/ext/{ggml → sources/ggml}/src/ggml-cpu/cpu-feats-x86.cpp +5 -1
  205. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +6431 -0
  206. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-impl.h +163 -41
  207. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.c +4029 -1117
  208. data/ext/sources/ggml/src/ggml-cpu/ggml-cpu.c +3510 -0
  209. data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu.cpp +67 -18
  210. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.cpp +337 -0
  211. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kernels.h +95 -0
  212. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +482 -0
  213. data/ext/sources/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  214. data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.cpp +3544 -0
  215. data/ext/sources/ggml/src/ggml-cpu/llamafile/sgemm.h +14 -0
  216. data/ext/sources/ggml/src/ggml-cpu/ops.cpp +8903 -0
  217. data/ext/sources/ggml/src/ggml-cpu/ops.h +110 -0
  218. data/ext/sources/ggml/src/ggml-cpu/simd-mappings.h +892 -0
  219. data/ext/sources/ggml/src/ggml-cpu/unary-ops.cpp +186 -0
  220. data/ext/sources/ggml/src/ggml-cpu/unary-ops.h +28 -0
  221. data/ext/sources/ggml/src/ggml-cpu/vec.cpp +252 -0
  222. data/ext/sources/ggml/src/ggml-cpu/vec.h +818 -0
  223. data/ext/sources/ggml/src/ggml-cuda/CMakeLists.txt +184 -0
  224. data/ext/sources/ggml/src/ggml-cuda/acc.cu +61 -0
  225. data/ext/sources/ggml/src/ggml-cuda/acc.cuh +5 -0
  226. data/ext/sources/ggml/src/ggml-cuda/arange.cu +34 -0
  227. data/ext/sources/ggml/src/ggml-cuda/arange.cuh +5 -0
  228. data/ext/sources/ggml/src/ggml-cuda/argmax.cu +91 -0
  229. data/ext/sources/ggml/src/ggml-cuda/argmax.cuh +3 -0
  230. data/ext/sources/ggml/src/ggml-cuda/argsort.cu +104 -0
  231. data/ext/sources/ggml/src/ggml-cuda/argsort.cuh +3 -0
  232. data/ext/sources/ggml/src/ggml-cuda/binbcast.cu +363 -0
  233. data/ext/sources/ggml/src/ggml-cuda/binbcast.cuh +9 -0
  234. data/ext/sources/ggml/src/ggml-cuda/clamp.cu +45 -0
  235. data/ext/sources/ggml/src/ggml-cuda/clamp.cuh +5 -0
  236. data/ext/sources/ggml/src/ggml-cuda/common.cuh +828 -0
  237. data/ext/sources/ggml/src/ggml-cuda/concat.cu +221 -0
  238. data/ext/sources/ggml/src/ggml-cuda/concat.cuh +5 -0
  239. data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cu +89 -0
  240. data/ext/sources/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  241. data/ext/sources/ggml/src/ggml-cuda/convert.cu +730 -0
  242. data/ext/sources/ggml/src/ggml-cuda/convert.cuh +26 -0
  243. data/ext/sources/ggml/src/ggml-cuda/count-equal.cu +64 -0
  244. data/ext/sources/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  245. data/ext/sources/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  246. data/ext/sources/ggml/src/ggml-cuda/cpy.cu +705 -0
  247. data/ext/sources/ggml/src/ggml-cuda/cpy.cuh +11 -0
  248. data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cu +189 -0
  249. data/ext/sources/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  250. data/ext/sources/ggml/src/ggml-cuda/dequantize.cuh +103 -0
  251. data/ext/sources/ggml/src/ggml-cuda/diagmask.cu +40 -0
  252. data/ext/sources/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  253. data/ext/sources/ggml/src/ggml-cuda/fattn-common.cuh +881 -0
  254. data/ext/sources/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1471 -0
  255. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cu +357 -0
  256. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f16.cuh +3 -0
  257. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cu +365 -0
  258. data/ext/sources/ggml/src/ggml-cuda/fattn-tile-f32.cuh +3 -0
  259. data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f16.cuh +482 -0
  260. data/ext/sources/ggml/src/ggml-cuda/fattn-vec-f32.cuh +472 -0
  261. data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cu +634 -0
  262. data/ext/sources/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +3 -0
  263. data/ext/sources/ggml/src/ggml-cuda/fattn.cu +346 -0
  264. data/ext/sources/ggml/src/ggml-cuda/fattn.cuh +3 -0
  265. data/ext/sources/ggml/src/ggml-cuda/getrows.cu +275 -0
  266. data/ext/sources/ggml/src/ggml-cuda/getrows.cuh +15 -0
  267. data/ext/sources/ggml/src/ggml-cuda/ggml-cuda.cu +3505 -0
  268. data/ext/sources/ggml/src/ggml-cuda/gla.cu +93 -0
  269. data/ext/sources/ggml/src/ggml-cuda/gla.cuh +3 -0
  270. data/ext/sources/ggml/src/ggml-cuda/im2col.cu +103 -0
  271. data/ext/sources/ggml/src/ggml-cuda/im2col.cuh +5 -0
  272. data/ext/sources/ggml/src/ggml-cuda/mma.cuh +396 -0
  273. data/ext/sources/ggml/src/ggml-cuda/mmq.cu +324 -0
  274. data/ext/sources/ggml/src/ggml-cuda/mmq.cuh +3217 -0
  275. data/ext/sources/ggml/src/ggml-cuda/mmv.cu +336 -0
  276. data/ext/sources/ggml/src/ggml-cuda/mmv.cuh +12 -0
  277. data/ext/sources/ggml/src/ggml-cuda/mmvq.cu +595 -0
  278. data/ext/sources/ggml/src/ggml-cuda/mmvq.cuh +12 -0
  279. data/ext/sources/ggml/src/ggml-cuda/norm.cu +458 -0
  280. data/ext/sources/ggml/src/ggml-cuda/norm.cuh +11 -0
  281. data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  282. data/ext/sources/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  283. data/ext/sources/ggml/src/ggml-cuda/out-prod.cu +68 -0
  284. data/ext/sources/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  285. data/ext/sources/ggml/src/ggml-cuda/pad.cu +49 -0
  286. data/ext/sources/ggml/src/ggml-cuda/pad.cuh +5 -0
  287. data/ext/sources/ggml/src/ggml-cuda/pool2d.cu +94 -0
  288. data/ext/sources/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  289. data/ext/sources/ggml/src/ggml-cuda/quantize.cu +190 -0
  290. data/ext/sources/ggml/src/ggml-cuda/quantize.cuh +27 -0
  291. data/ext/sources/ggml/src/ggml-cuda/rope.cu +456 -0
  292. data/ext/sources/ggml/src/ggml-cuda/rope.cuh +7 -0
  293. data/ext/sources/ggml/src/ggml-cuda/scale.cu +31 -0
  294. data/ext/sources/ggml/src/ggml-cuda/scale.cuh +5 -0
  295. data/ext/sources/ggml/src/ggml-cuda/softmax.cu +283 -0
  296. data/ext/sources/ggml/src/ggml-cuda/softmax.cuh +7 -0
  297. data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cu +148 -0
  298. data/ext/sources/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  299. data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cu +153 -0
  300. data/ext/sources/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  301. data/ext/sources/ggml/src/ggml-cuda/sum.cu +45 -0
  302. data/ext/sources/ggml/src/ggml-cuda/sum.cuh +5 -0
  303. data/ext/sources/ggml/src/ggml-cuda/sumrows.cu +39 -0
  304. data/ext/sources/ggml/src/ggml-cuda/sumrows.cuh +5 -0
  305. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
  306. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +10 -0
  307. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  308. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
  309. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +10 -0
  310. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
  311. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +10 -0
  312. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +10 -0
  313. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  314. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
  315. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
  316. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
  317. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +10 -0
  318. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +10 -0
  319. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  320. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  321. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
  322. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +10 -0
  323. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +10 -0
  324. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu +5 -0
  325. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu +5 -0
  326. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu +5 -0
  327. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu +5 -0
  328. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu +5 -0
  329. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu +5 -0
  330. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu +5 -0
  331. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu +5 -0
  332. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu +5 -0
  333. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu +5 -0
  334. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu +5 -0
  335. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu +5 -0
  336. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu +5 -0
  337. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu +5 -0
  338. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu +5 -0
  339. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu +5 -0
  340. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu +5 -0
  341. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu +5 -0
  342. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu +5 -0
  343. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu +5 -0
  344. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu +5 -0
  345. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu +5 -0
  346. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu +5 -0
  347. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu +5 -0
  348. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu +5 -0
  349. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu +5 -0
  350. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu +5 -0
  351. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu +5 -0
  352. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu +5 -0
  353. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu +5 -0
  354. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu +5 -0
  355. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu +5 -0
  356. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu +5 -0
  357. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu +5 -0
  358. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu +5 -0
  359. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu +5 -0
  360. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu +5 -0
  361. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu +5 -0
  362. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu +5 -0
  363. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu +5 -0
  364. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu +5 -0
  365. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu +5 -0
  366. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu +5 -0
  367. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu +5 -0
  368. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu +5 -0
  369. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu +5 -0
  370. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu +5 -0
  371. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu +5 -0
  372. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu +5 -0
  373. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu +5 -0
  374. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu +5 -0
  375. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu +5 -0
  376. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu +5 -0
  377. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu +5 -0
  378. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu +5 -0
  379. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu +5 -0
  380. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu +5 -0
  381. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu +5 -0
  382. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu +5 -0
  383. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu +5 -0
  384. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu +5 -0
  385. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu +5 -0
  386. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu +5 -0
  387. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu +5 -0
  388. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu +5 -0
  389. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu +5 -0
  390. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu +5 -0
  391. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu +5 -0
  392. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu +5 -0
  393. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu +5 -0
  394. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu +5 -0
  395. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu +5 -0
  396. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu +5 -0
  397. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu +5 -0
  398. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu +5 -0
  399. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu +5 -0
  400. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu +5 -0
  401. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu +5 -0
  402. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu +5 -0
  403. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu +5 -0
  404. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu +5 -0
  405. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu +5 -0
  406. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu +5 -0
  407. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu +5 -0
  408. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu +5 -0
  409. data/ext/sources/ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu +5 -0
  410. data/ext/sources/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +78 -0
  411. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  412. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  413. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  414. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  415. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  416. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  417. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  418. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  419. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  420. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  421. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  422. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  423. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  424. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  425. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  426. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  427. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  428. data/ext/sources/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  429. data/ext/sources/ggml/src/ggml-cuda/tsembd.cu +47 -0
  430. data/ext/sources/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  431. data/ext/sources/ggml/src/ggml-cuda/unary.cu +289 -0
  432. data/ext/sources/ggml/src/ggml-cuda/unary.cuh +59 -0
  433. data/ext/sources/ggml/src/ggml-cuda/upscale.cu +51 -0
  434. data/ext/sources/ggml/src/ggml-cuda/upscale.cuh +5 -0
  435. data/ext/sources/ggml/src/ggml-cuda/vecdotq.cuh +1135 -0
  436. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/cuda.h +1 -0
  437. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/hip.h +57 -0
  438. data/ext/{ggml → sources/ggml}/src/ggml-cuda/vendors/musa.h +7 -1
  439. data/ext/sources/ggml/src/ggml-cuda/wkv.cu +199 -0
  440. data/ext/sources/ggml/src/ggml-cuda/wkv.cuh +7 -0
  441. data/ext/sources/ggml/src/ggml-hip/CMakeLists.txt +131 -0
  442. data/ext/{ggml → sources/ggml}/src/ggml-impl.h +64 -19
  443. data/ext/sources/ggml/src/ggml-kompute/CMakeLists.txt +166 -0
  444. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/common.comp +112 -0
  445. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_add.comp +58 -0
  446. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_addrow.comp +25 -0
  447. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f16.comp +52 -0
  448. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f16_f32.comp +52 -0
  449. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f16.comp +52 -0
  450. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_cpy_f32_f32.comp +52 -0
  451. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_diagmask.comp +30 -0
  452. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_gelu.comp +22 -0
  453. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows.comp +17 -0
  454. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f16.comp +31 -0
  455. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_f32.comp +31 -0
  456. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_0.comp +38 -0
  457. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q4_1.comp +39 -0
  458. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_getrows_q6_k.comp +44 -0
  459. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul.comp +52 -0
  460. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_f16.comp +69 -0
  461. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_mat_f32.comp +51 -0
  462. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_0.comp +33 -0
  463. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_1.comp +35 -0
  464. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q4_k.comp +140 -0
  465. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q6_k.comp +106 -0
  466. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mat_q8_0.comp +73 -0
  467. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n.comp +52 -0
  468. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_mul_mv_q_n_pre.comp +28 -0
  469. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_norm.comp +84 -0
  470. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_relu.comp +21 -0
  471. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rmsnorm.comp +53 -0
  472. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f16.comp +52 -0
  473. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_neox_f32.comp +52 -0
  474. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f16.comp +52 -0
  475. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_rope_norm_f32.comp +52 -0
  476. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale.comp +19 -0
  477. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_scale_8.comp +23 -0
  478. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_silu.comp +22 -0
  479. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/op_softmax.comp +72 -0
  480. data/ext/sources/ggml/src/ggml-kompute/kompute-shaders/rope_common.comp +71 -0
  481. data/ext/sources/ggml/src/ggml-metal/CMakeLists.txt +120 -0
  482. data/ext/sources/ggml/src/ggml-metal/ggml-metal-impl.h +622 -0
  483. data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.m +2178 -1064
  484. data/ext/{ggml → sources/ggml}/src/ggml-metal/ggml-metal.metal +1575 -1218
  485. data/ext/sources/ggml/src/ggml-musa/CMakeLists.txt +113 -0
  486. data/ext/sources/ggml/src/ggml-musa/mudnn.cu +112 -0
  487. data/ext/sources/ggml/src/ggml-musa/mudnn.cuh +12 -0
  488. data/ext/sources/ggml/src/ggml-opencl/CMakeLists.txt +96 -0
  489. data/ext/sources/ggml/src/ggml-opencl/ggml-opencl.cpp +5124 -0
  490. data/ext/sources/ggml/src/ggml-opencl/kernels/add.cl +83 -0
  491. data/ext/sources/ggml/src/ggml-opencl/kernels/clamp.cl +20 -0
  492. data/ext/sources/ggml/src/ggml-opencl/kernels/cpy.cl +184 -0
  493. data/ext/sources/ggml/src/ggml-opencl/kernels/cvt.cl +118 -0
  494. data/ext/sources/ggml/src/ggml-opencl/kernels/diag_mask_inf.cl +58 -0
  495. data/ext/sources/ggml/src/ggml-opencl/kernels/embed_kernel.py +26 -0
  496. data/ext/sources/ggml/src/ggml-opencl/kernels/gelu.cl +62 -0
  497. data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle.cl +268 -0
  498. data/ext/sources/ggml/src/ggml-opencl/kernels/gemv_noshuffle_general.cl +274 -0
  499. data/ext/sources/ggml/src/ggml-opencl/kernels/get_rows.cl +163 -0
  500. data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f16.cl +57 -0
  501. data/ext/sources/ggml/src/ggml-opencl/kernels/im2col_f32.cl +57 -0
  502. data/ext/sources/ggml/src/ggml-opencl/kernels/mul.cl +79 -0
  503. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mat_Ab_Bi_8x4.cl +139 -0
  504. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl +118 -0
  505. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl +118 -0
  506. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl +94 -0
  507. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +84 -0
  508. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +118 -0
  509. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  510. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  511. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  512. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  513. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  514. data/ext/sources/ggml/src/ggml-opencl/kernels/mul_mv_q6_k.cl +190 -0
  515. data/ext/sources/ggml/src/ggml-opencl/kernels/norm.cl +81 -0
  516. data/ext/sources/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  517. data/ext/sources/ggml/src/ggml-opencl/kernels/rms_norm.cl +96 -0
  518. data/ext/sources/ggml/src/ggml-opencl/kernels/rope.cl +721 -0
  519. data/ext/sources/ggml/src/ggml-opencl/kernels/scale.cl +16 -0
  520. data/ext/sources/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  521. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +87 -0
  522. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +87 -0
  523. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f16.cl +86 -0
  524. data/ext/sources/ggml/src/ggml-opencl/kernels/softmax_f32.cl +86 -0
  525. data/ext/sources/ggml/src/ggml-opencl/kernels/transpose.cl +84 -0
  526. data/ext/{ggml → sources/ggml}/src/ggml-opt.cpp +373 -190
  527. data/ext/{ggml → sources/ggml}/src/ggml-quants.c +114 -120
  528. data/ext/sources/ggml/src/ggml-rpc/CMakeLists.txt +9 -0
  529. data/ext/{ggml → sources/ggml}/src/ggml-rpc/ggml-rpc.cpp +480 -73
  530. data/ext/sources/ggml/src/ggml-sycl/CMakeLists.txt +189 -0
  531. data/ext/sources/ggml/src/ggml-sycl/backend.hpp +37 -0
  532. data/ext/sources/ggml/src/ggml-sycl/binbcast.cpp +345 -0
  533. data/ext/sources/ggml/src/ggml-sycl/binbcast.hpp +39 -0
  534. data/ext/{ggml → sources/ggml}/src/ggml-sycl/common.cpp +20 -32
  535. data/ext/sources/ggml/src/ggml-sycl/common.hpp +589 -0
  536. data/ext/{ggml → sources/ggml}/src/ggml-sycl/concat.cpp +32 -33
  537. data/ext/sources/ggml/src/ggml-sycl/concat.hpp +20 -0
  538. data/ext/{ggml → sources/ggml}/src/ggml-sycl/conv.cpp +4 -2
  539. data/ext/sources/ggml/src/ggml-sycl/conv.hpp +20 -0
  540. data/ext/{ggml → sources/ggml}/src/ggml-sycl/convert.cpp +104 -28
  541. data/ext/sources/ggml/src/ggml-sycl/convert.hpp +34 -0
  542. data/ext/sources/ggml/src/ggml-sycl/cpy.cpp +700 -0
  543. data/ext/sources/ggml/src/ggml-sycl/cpy.hpp +11 -0
  544. data/ext/sources/ggml/src/ggml-sycl/dequantize.hpp +791 -0
  545. data/ext/{ggml → sources/ggml}/src/ggml-sycl/dmmv.cpp +156 -17
  546. data/ext/sources/ggml/src/ggml-sycl/dmmv.hpp +27 -0
  547. data/ext/sources/ggml/src/ggml-sycl/dpct/helper.hpp +2957 -0
  548. data/ext/sources/ggml/src/ggml-sycl/element_wise.cpp +1511 -0
  549. data/ext/sources/ggml/src/ggml-sycl/element_wise.hpp +75 -0
  550. data/ext/sources/ggml/src/ggml-sycl/gemm.hpp +99 -0
  551. data/ext/sources/ggml/src/ggml-sycl/getrows.cpp +309 -0
  552. data/ext/sources/ggml/src/ggml-sycl/getrows.hpp +20 -0
  553. data/ext/{ggml → sources/ggml}/src/ggml-sycl/ggml-sycl.cpp +1004 -1240
  554. data/ext/sources/ggml/src/ggml-sycl/gla.cpp +106 -0
  555. data/ext/sources/ggml/src/ggml-sycl/gla.hpp +8 -0
  556. data/ext/sources/ggml/src/ggml-sycl/im2col.cpp +136 -0
  557. data/ext/sources/ggml/src/ggml-sycl/im2col.hpp +21 -0
  558. data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmq.cpp +0 -1
  559. data/ext/sources/ggml/src/ggml-sycl/mmq.hpp +33 -0
  560. data/ext/{ggml → sources/ggml}/src/ggml-sycl/mmvq.cpp +261 -166
  561. data/ext/sources/ggml/src/ggml-sycl/mmvq.hpp +27 -0
  562. data/ext/{ggml → sources/ggml}/src/ggml-sycl/norm.cpp +204 -81
  563. data/ext/sources/ggml/src/ggml-sycl/norm.hpp +26 -0
  564. data/ext/{ggml → sources/ggml}/src/ggml-sycl/outprod.cpp +8 -17
  565. data/ext/sources/ggml/src/ggml-sycl/outprod.hpp +10 -0
  566. data/ext/sources/ggml/src/ggml-sycl/presets.hpp +74 -0
  567. data/ext/sources/ggml/src/ggml-sycl/quants.hpp +83 -0
  568. data/ext/sources/ggml/src/ggml-sycl/rope.cpp +361 -0
  569. data/ext/sources/ggml/src/ggml-sycl/rope.hpp +20 -0
  570. data/ext/{ggml → sources/ggml}/src/ggml-sycl/softmax.cpp +35 -25
  571. data/ext/sources/ggml/src/ggml-sycl/softmax.hpp +20 -0
  572. data/ext/sources/ggml/src/ggml-sycl/sycl_hw.cpp +13 -0
  573. data/ext/sources/ggml/src/ggml-sycl/sycl_hw.hpp +23 -0
  574. data/ext/{ggml → sources/ggml}/src/ggml-sycl/tsembd.cpp +3 -3
  575. data/ext/sources/ggml/src/ggml-sycl/tsembd.hpp +20 -0
  576. data/ext/sources/ggml/src/ggml-sycl/vecdotq.hpp +1215 -0
  577. data/ext/sources/ggml/src/ggml-sycl/wkv.cpp +293 -0
  578. data/ext/sources/ggml/src/ggml-sycl/wkv.hpp +10 -0
  579. data/ext/sources/ggml/src/ggml-vulkan/CMakeLists.txt +196 -0
  580. data/ext/sources/ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in +15 -0
  581. data/ext/{ggml → sources/ggml}/src/ggml-vulkan/ggml-vulkan.cpp +3130 -1087
  582. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +39 -0
  583. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/acc.comp +29 -0
  584. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/add.comp +29 -0
  585. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp +51 -0
  586. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp +69 -0
  587. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp +17 -0
  588. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/concat.comp +41 -0
  589. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp +49 -0
  590. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp +105 -0
  591. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy.comp +23 -0
  592. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp +51 -0
  593. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +242 -0
  594. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/cos.comp +17 -0
  595. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp +31 -0
  596. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp +20 -0
  597. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp +462 -0
  598. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.comp +699 -0
  599. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.comp +13 -0
  600. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp +42 -0
  601. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp +35 -0
  602. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +44 -0
  603. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +43 -0
  604. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +48 -0
  605. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +39 -0
  606. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +49 -0
  607. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp +32 -0
  608. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp +34 -0
  609. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp +34 -0
  610. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp +42 -0
  611. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp +30 -0
  612. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp +32 -0
  613. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp +68 -0
  614. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp +34 -0
  615. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp +35 -0
  616. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp +70 -0
  617. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp +33 -0
  618. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp +31 -0
  619. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp +34 -0
  620. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/div.comp +27 -0
  621. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +337 -0
  622. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.comp +162 -0
  623. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +360 -0
  624. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +267 -0
  625. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp +59 -0
  626. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu.comp +25 -0
  627. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/gelu_quick.comp +23 -0
  628. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.comp +64 -0
  629. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_head.comp +9 -0
  630. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.comp +76 -0
  631. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp +33 -0
  632. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp +41 -0
  633. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp +66 -0
  634. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp +100 -0
  635. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  636. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp +22 -0
  637. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  638. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  639. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +169 -0
  640. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.comp +118 -0
  641. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +82 -0
  642. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +79 -0
  643. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  644. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +87 -0
  645. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  646. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +90 -0
  647. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  648. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +118 -0
  649. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +154 -0
  650. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +130 -0
  651. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  652. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +136 -0
  653. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +167 -0
  654. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  655. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +868 -0
  656. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +441 -0
  657. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +442 -0
  658. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.comp +99 -0
  659. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  660. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  661. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +28 -0
  662. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  663. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +77 -0
  664. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/relu.comp +21 -0
  665. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  666. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  667. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +52 -0
  668. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  669. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_head.comp +58 -0
  670. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +60 -0
  671. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp +43 -0
  672. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp +43 -0
  673. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp +47 -0
  674. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp +24 -0
  675. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sigmoid.comp +20 -0
  676. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu.comp +22 -0
  677. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp +26 -0
  678. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sin.comp +17 -0
  679. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp +173 -0
  680. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp +50 -0
  681. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/square.comp +17 -0
  682. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp +29 -0
  683. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp +37 -0
  684. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/tanh.comp +20 -0
  685. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_bfloat16_support.comp +7 -0
  686. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat2_support.comp +7 -0
  687. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_coopmat_support.comp +7 -0
  688. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/test_integer_dot_support.comp +7 -0
  689. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp +41 -0
  690. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +1373 -0
  691. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp +36 -0
  692. data/ext/{ggml → sources/ggml}/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +193 -35
  693. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp +87 -0
  694. data/ext/sources/ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp +91 -0
  695. data/ext/{ggml → sources/ggml}/src/ggml.c +676 -1820
  696. data/ext/sources/ggml/src/gguf.cpp +1330 -0
  697. data/ext/{include → sources/include}/whisper.h +68 -2
  698. data/ext/sources/src/CMakeLists.txt +143 -0
  699. data/ext/{src → sources/src}/coreml/whisper-decoder-impl.h +27 -15
  700. data/ext/{src → sources/src}/coreml/whisper-decoder-impl.m +35 -10
  701. data/ext/{src → sources/src}/coreml/whisper-encoder-impl.h +21 -9
  702. data/ext/{src → sources/src}/coreml/whisper-encoder-impl.m +28 -3
  703. data/ext/sources/src/coreml/whisper-encoder.mm +73 -0
  704. data/ext/sources/src/whisper-arch.h +197 -0
  705. data/ext/{src → sources/src}/whisper.cpp +1905 -374
  706. data/ext/sources/tests/CMakeLists.txt +105 -0
  707. data/ext/sources/tests/earnings21/eval.mk +58 -0
  708. data/ext/sources/tests/earnings21/eval.py +68 -0
  709. data/ext/sources/tests/earnings21/normalizers/__init__.py +2 -0
  710. data/ext/sources/tests/earnings21/normalizers/basic.py +80 -0
  711. data/ext/sources/tests/earnings21/normalizers/english.json +1741 -0
  712. data/ext/sources/tests/earnings21/normalizers/english.py +550 -0
  713. data/ext/sources/tests/earnings21/requirements.txt +6 -0
  714. data/ext/sources/tests/en-0-ref.txt +1 -0
  715. data/ext/sources/tests/en-1-ref.txt +1 -0
  716. data/ext/sources/tests/en-2-ref.txt +1 -0
  717. data/ext/sources/tests/es-0-ref.txt +1 -0
  718. data/ext/sources/tests/librispeech/eval.mk +39 -0
  719. data/ext/sources/tests/librispeech/eval.py +47 -0
  720. data/ext/sources/tests/librispeech/normalizers/__init__.py +2 -0
  721. data/ext/sources/tests/librispeech/normalizers/basic.py +80 -0
  722. data/ext/sources/tests/librispeech/normalizers/english.json +1741 -0
  723. data/ext/sources/tests/librispeech/normalizers/english.py +550 -0
  724. data/ext/sources/tests/librispeech/requirements.txt +6 -0
  725. data/ext/sources/tests/run-tests.sh +130 -0
  726. data/ext/sources/tests/test-c.c +3 -0
  727. data/ext/sources/tests/test-vad-full.cpp +54 -0
  728. data/ext/sources/tests/test-vad.cpp +83 -0
  729. data/ext/sources/tests/test-whisper.js +58 -0
  730. data/extsources.rb +33 -5
  731. data/lib/whisper/model/uri.rb +149 -128
  732. data/sig/whisper.rbs +480 -0
  733. data/tests/helper.rb +28 -0
  734. data/tests/test_callback.rb +45 -3
  735. data/tests/test_error.rb +2 -2
  736. data/tests/test_model.rb +38 -0
  737. data/tests/test_package.rb +18 -3
  738. data/tests/test_params.rb +145 -8
  739. data/tests/test_segment.rb +10 -19
  740. data/tests/test_vad.rb +19 -0
  741. data/tests/test_vad_params.rb +103 -0
  742. data/tests/test_whisper.rb +37 -37
  743. data/whispercpp.gemspec +5 -4
  744. metadata +766 -111
  745. data/ext/cpu.mk +0 -9
  746. data/ext/examples/dr_wav.h +0 -8815
  747. data/ext/ggml/src/ggml-cann/aclnn_ops.h +0 -592
  748. data/ext/ggml/src/ggml-cpu/ggml-cpu-aarch64.cpp +0 -4262
  749. data/ext/ggml/src/ggml-cpu/ggml-cpu.c +0 -14123
  750. data/ext/ggml/src/ggml-cpu/llamafile/sgemm.cpp +0 -1884
  751. data/ext/ggml/src/ggml-cpu/llamafile/sgemm.h +0 -14
  752. data/ext/ggml/src/ggml-metal/ggml-metal-impl.h +0 -288
  753. data/ext/ggml/src/ggml-sycl/element_wise.cpp +0 -1030
  754. data/ext/ggml/src/ggml-sycl/im2col.cpp +0 -126
  755. data/ext/ggml/src/ggml-sycl/rope.cpp +0 -276
  756. data/ext/ggml/src/ggml-sycl/wkv6.cpp +0 -141
  757. data/ext/metal-embed.mk +0 -17
  758. data/ext/metal.mk +0 -6
  759. data/ext/ruby_whisper.cpp +0 -1909
  760. data/ext/scripts/get-flags.mk +0 -38
  761. data/lib/whisper.rb +0 -2
  762. /data/ext/{ggml → sources/ggml}/include/ggml-blas.h +0 -0
  763. /data/ext/{ggml → sources/ggml}/include/ggml-cann.h +0 -0
  764. /data/ext/{ggml → sources/ggml}/include/ggml-cuda.h +0 -0
  765. /data/ext/{ggml → sources/ggml}/include/ggml-kompute.h +0 -0
  766. /data/ext/{ggml → sources/ggml}/include/ggml-opencl.h +0 -0
  767. /data/ext/{ggml → sources/ggml}/include/ggml-sycl.h +0 -0
  768. /data/ext/{ggml → sources/ggml}/src/ggml-amx/common.h +0 -0
  769. /data/ext/{ggml → sources/ggml}/src/ggml-amx/ggml-amx.cpp +0 -0
  770. /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.cpp +0 -0
  771. /data/ext/{ggml → sources/ggml}/src/ggml-amx/mmq.h +0 -0
  772. /data/ext/{ggml → sources/ggml}/src/ggml-blas/ggml-blas.cpp +0 -0
  773. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/ascendc_kernels.h +0 -0
  774. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f16.cpp +0 -0
  775. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_f32.cpp +0 -0
  776. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q4_0.cpp +0 -0
  777. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/get_row_q8_0.cpp +0 -0
  778. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f16_q8_0.cpp +0 -0
  779. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_f32_q8_0.cpp +0 -0
  780. /data/ext/{ggml → sources/ggml}/src/ggml-cann/kernels/quantize_float_to_q4_0.cpp +0 -0
  781. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/amx.h +0 -0
  782. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/common.h +0 -0
  783. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.cpp +0 -0
  784. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/amx/mmq.h +0 -0
  785. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-aarch64.h +0 -0
  786. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.cpp +0 -0
  787. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-hbm.h +0 -0
  788. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-quants.h +0 -0
  789. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.cpp +0 -0
  790. /data/ext/{ggml → sources/ggml}/src/ggml-cpu/ggml-cpu-traits.h +0 -0
  791. /data/ext/{ggml → sources/ggml}/src/ggml-kompute/ggml-kompute.cpp +0 -0
  792. /data/ext/{ggml → sources/ggml}/src/ggml-quants.h +0 -0
  793. /data/ext/{ggml → sources/ggml}/src/ggml-threading.cpp +0 -0
  794. /data/ext/{ggml → sources/ggml}/src/ggml-threading.h +0 -0
  795. /data/ext/{src → sources/src}/coreml/whisper-encoder.h +0 -0
  796. /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.cpp +0 -0
  797. /data/ext/{src → sources/src}/openvino/whisper-openvino-encoder.h +0 -0
@@ -0,0 +1,2575 @@
1
+ #include "llama-sampling.h"
2
+
3
+ #include "llama-impl.h"
4
+ #include "llama-vocab.h"
5
+ #include "llama-grammar.h"
6
+
7
+ #include <algorithm>
8
+ #include <cassert>
9
+ #include <cfloat>
10
+ #include <chrono>
11
+ #include <cmath>
12
+ #include <cstdlib>
13
+ #include <cstring>
14
+ #include <ctime>
15
+ #include <numeric>
16
+ #include <random>
17
+ #include <unordered_map>
18
+ #include <stdexcept>
19
+
20
+ // the ring buffer works similarly to std::deque, but with a fixed capacity
21
+ template<typename T>
22
+ struct ring_buffer {
23
+ ring_buffer(size_t cap) : capacity(cap), data(cap) {}
24
+
25
+ T & front() {
26
+ if (sz == 0) {
27
+ throw std::runtime_error("ring buffer is empty");
28
+ }
29
+ return data[first];
30
+ }
31
+
32
+ const T & front() const {
33
+ if (sz == 0) {
34
+ throw std::runtime_error("ring buffer is empty");
35
+ }
36
+ return data[first];
37
+ }
38
+
39
+ T & back() {
40
+ if (sz == 0) {
41
+ throw std::runtime_error("ring buffer is empty");
42
+ }
43
+ return data[pos];
44
+ }
45
+
46
+ const T & back() const {
47
+ if (sz == 0) {
48
+ throw std::runtime_error("ring buffer is empty");
49
+ }
50
+ return data[pos];
51
+ }
52
+
53
+ void push_back(const T & value) {
54
+ if (capacity == 0) {
55
+ throw std::runtime_error("ring buffer: capacity is zero");
56
+ }
57
+
58
+ if (sz == capacity) {
59
+ // advance the start when buffer is full
60
+ first = (first + 1) % capacity;
61
+ } else {
62
+ sz++;
63
+ }
64
+ data[pos] = value;
65
+ pos = (pos + 1) % capacity;
66
+ }
67
+
68
+ T pop_front() {
69
+ if (sz == 0) {
70
+ throw std::runtime_error("ring buffer is empty");
71
+ }
72
+ T value = data[first];
73
+ first = (first + 1) % capacity;
74
+ sz--;
75
+ return value;
76
+ }
77
+
78
+ //T & operator[](size_t i) {
79
+ // if (i >= sz) {
80
+ // throw std::runtime_error("ring buffer: index out of bounds");
81
+ // }
82
+ // return data[(first + i) % capacity];
83
+ //}
84
+
85
+ //const T & at(size_t i) const {
86
+ // if (i >= sz) {
87
+ // throw std::runtime_error("ring buffer: index out of bounds");
88
+ // }
89
+ // return data[(first + i) % capacity];
90
+ //}
91
+
92
+ const T & rat(size_t i) const {
93
+ if (i >= sz) {
94
+ throw std::runtime_error("ring buffer: index out of bounds");
95
+ }
96
+ return data[(first + sz - i - 1) % capacity];
97
+ }
98
+
99
+ std::vector<T> to_vector() const {
100
+ std::vector<T> result;
101
+ result.reserve(sz);
102
+ for (size_t i = 0; i < sz; i++) {
103
+ result.push_back(data[(first + i) % capacity]);
104
+ }
105
+ return result;
106
+ }
107
+
108
+ void clear() {
109
+ // here only reset the status of the buffer
110
+ sz = 0;
111
+ first = 0;
112
+ pos = 0;
113
+ }
114
+
115
+ bool empty() const {
116
+ return sz == 0;
117
+ }
118
+
119
+ size_t size() const {
120
+ return sz;
121
+ }
122
+
123
+ size_t capacity = 0;
124
+ size_t sz = 0;
125
+ size_t first = 0;
126
+ size_t pos = 0;
127
+
128
+ std::vector<T> data;
129
+ };
130
+
131
+ static int llama_sample_dist(llama_token_data_array * cur_p, std::mt19937 & rng) {
132
+ // iterator for the probabilities
133
+ #ifdef __GNUC__
134
+ #pragma GCC diagnostic push
135
+ #pragma GCC diagnostic ignored "-Wunused-local-typedefs"
136
+ #endif
137
+
138
+ struct probs_iterator {
139
+ typedef std::input_iterator_tag iterator_category;
140
+ typedef float value_type;
141
+ typedef float * pointer;
142
+ typedef float & reference;
143
+ typedef ptrdiff_t difference_type;
144
+
145
+ const llama_token_data * data;
146
+
147
+ bool operator==(const probs_iterator & other) const { return data == other.data; }
148
+ bool operator!=(const probs_iterator & other) const { return data != other.data; }
149
+ const float & operator*() const { return data->p; }
150
+ probs_iterator & operator++() { ++data; return *this; }
151
+ probs_iterator operator++(int) { probs_iterator tmp = *this; ++data; return tmp; }
152
+ };
153
+
154
+ #ifdef __GNUC__
155
+ #pragma GCC diagnostic pop
156
+ #endif
157
+
158
+ std::discrete_distribution<int> dist(probs_iterator{cur_p->data}, probs_iterator{cur_p->data + cur_p->size});
159
+
160
+ return dist(rng);
161
+ }
162
+
163
+ /*
164
+ static void llama_log_softmax(float * array, size_t size) {
165
+ float max_l = *std::max_element(array, array + size);
166
+ float sum = 0.f;
167
+ for (size_t i = 0; i < size; ++i) {
168
+ float p = expf(array[i] - max_l);
169
+ sum += p;
170
+ array[i] = p;
171
+ }
172
+
173
+ for (size_t i = 0; i < size; ++i) {
174
+ array[i] = logf(array[i] / sum);
175
+ }
176
+ }
177
+ */
178
+
179
+ static void llama_sampler_temp_impl(llama_token_data_array * cur_p, float temp) {
180
+ if (temp <= 0.0f) {
181
+ // find the token with the highest logit and set the rest to -inf
182
+ size_t max_i = 0;
183
+ float max_l = cur_p->data[0].logit;
184
+
185
+ for (size_t i = 1; i < cur_p->size; ++i) {
186
+ if (cur_p->data[i ].logit > max_l) {
187
+ cur_p->data[max_i].logit = -INFINITY;
188
+ max_i = i;
189
+ max_l = cur_p->data[i].logit;
190
+ } else {
191
+ cur_p->data[i].logit = -INFINITY;
192
+ }
193
+ }
194
+
195
+ return;
196
+ }
197
+
198
+ for (size_t i = 0; i < cur_p->size; ++i) {
199
+ cur_p->data[i].logit /= temp;
200
+ }
201
+ }
202
+
203
+ static void llama_sampler_softmax_impl(llama_token_data_array * cur_p) {
204
+ GGML_ASSERT(cur_p->size > 0);
205
+
206
+ // Sort the logits in descending order
207
+ if (!cur_p->sorted) {
208
+ std::sort(cur_p->data, cur_p->data + cur_p->size, [](const llama_token_data & a, const llama_token_data & b) {
209
+ return a.logit > b.logit;
210
+ });
211
+ cur_p->sorted = true;
212
+ }
213
+
214
+ float max_l = cur_p->data[0].logit;
215
+ float cum_sum = 0.0f;
216
+
217
+ for (size_t i = 0; i < cur_p->size; ++i) {
218
+ float p = expf(cur_p->data[i].logit - max_l);
219
+ cur_p->data[i].p = p;
220
+ cum_sum += p;
221
+ }
222
+
223
+ for (size_t i = 0; i < cur_p->size; ++i) {
224
+ cur_p->data[i].p /= cum_sum;
225
+ }
226
+ }
227
+
228
+ static void llama_sampler_top_k_impl(llama_token_data_array * cur_p, int32_t k) {
229
+ // TODO: move bucket sort to separate function so that top_p/typical/softmax first is equally fast
230
+ // if (k >= (int32_t)cur_p->size) {
231
+ // return;
232
+ // }
233
+
234
+ if (k <= 0) {
235
+ return;
236
+ }
237
+
238
+ k = std::min(k, (int) cur_p->size);
239
+
240
+ // Sort scores in descending order
241
+ if (!cur_p->sorted) {
242
+ auto comp = [](const llama_token_data & a, const llama_token_data & b) {
243
+ return a.logit > b.logit;
244
+ };
245
+ if (k <= 128) {
246
+ std::partial_sort(cur_p->data, cur_p->data + k, cur_p->data + cur_p->size, comp);
247
+ } else {
248
+ constexpr int nbuckets = 128;
249
+ constexpr float bucket_low = -10.0f;
250
+ constexpr float bucket_high = 10.0f;
251
+ constexpr float bucket_scale = nbuckets/(bucket_high - bucket_low);
252
+ constexpr float bucket_inter = -bucket_low * bucket_scale;
253
+
254
+ std::vector<int> bucket_idx(cur_p->size);
255
+ std::vector<int> histo(nbuckets, 0);
256
+
257
+ for (int i = 0; i < (int)cur_p->size; ++i) {
258
+ const float val = cur_p->data[i].logit;
259
+ int ib = int(bucket_scale * val + bucket_inter); //nbuckets * (val - bucket_low) / (bucket_high - bucket_low);
260
+ ib = std::max(0, std::min(nbuckets - 1, ib));
261
+ bucket_idx[i] = ib;
262
+ ++histo[ib];
263
+ }
264
+ int nhave = 0;
265
+ int ib = nbuckets - 1;
266
+ for ( ; ib >= 0; --ib) {
267
+ nhave += histo[ib];
268
+ if (nhave >= k) {
269
+ break;
270
+ }
271
+ }
272
+ std::vector<llama_token_data> tmp_tokens(nhave);
273
+ auto * ptr = tmp_tokens.data();
274
+ std::vector<llama_token_data*> bucket_ptrs;
275
+ bucket_ptrs.reserve(nbuckets - ib);
276
+ for (int j = nbuckets - 1; j >= ib; --j) {
277
+ bucket_ptrs.push_back(ptr);
278
+ ptr += histo[j];
279
+ }
280
+ for (int i = 0; i < (int)cur_p->size; ++i) {
281
+ int j = bucket_idx[i];
282
+ if (j >= ib) {
283
+ *bucket_ptrs[nbuckets - 1 - j]++ = cur_p->data[i];
284
+ }
285
+ }
286
+
287
+ ptr = tmp_tokens.data();
288
+ int ndone = 0;
289
+ for (int j = nbuckets - 1; j > ib; --j) {
290
+ std::sort(ptr, ptr + histo[j], comp);
291
+ ptr += histo[j];
292
+ ndone += histo[j];
293
+ }
294
+ std::partial_sort(ptr, ptr + k - ndone, ptr + histo[ib], comp);
295
+
296
+ std::memcpy(cur_p->data, tmp_tokens.data(), k*sizeof(llama_token_data));
297
+
298
+ }
299
+ cur_p->sorted = true;
300
+ }
301
+
302
+ cur_p->size = k;
303
+ }
304
+
305
+ static uint32_t get_rng_seed(uint32_t seed) {
306
+ if (seed == LLAMA_DEFAULT_SEED) {
307
+ // use system clock if std::random_device is not a true RNG
308
+ static bool is_rd_prng = std::random_device().entropy() == 0;
309
+ if (is_rd_prng) {
310
+ return (uint32_t) std::chrono::system_clock::now().time_since_epoch().count();
311
+ }
312
+ std::random_device rd;
313
+ return rd();
314
+ }
315
+ return seed;
316
+ }
317
+
318
+ // llama_sampler API
319
+
320
+ struct llama_sampler * llama_sampler_init(const struct llama_sampler_i * iface, llama_sampler_context_t ctx) {
321
+ return new llama_sampler {
322
+ /* .iface = */ iface,
323
+ /* .ctx = */ ctx,
324
+ };
325
+ }
326
+
327
+ const char * llama_sampler_name(const struct llama_sampler * smpl) {
328
+ if (!smpl->iface) {
329
+ return "(null)";
330
+ }
331
+
332
+ return smpl->iface->name(smpl);
333
+ }
334
+
335
+ void llama_sampler_accept(struct llama_sampler * smpl, llama_token token) {
336
+ if (smpl->iface->accept) {
337
+ smpl->iface->accept(smpl, token);
338
+ }
339
+ }
340
+
341
+ void llama_sampler_apply(struct llama_sampler * smpl, struct llama_token_data_array * cur_p) {
342
+ GGML_ASSERT(smpl->iface->apply);
343
+ smpl->iface->apply(smpl, cur_p);
344
+ }
345
+
346
+ void llama_sampler_reset(struct llama_sampler * smpl) {
347
+ if (smpl->iface->reset) {
348
+ smpl->iface->reset(smpl);
349
+ }
350
+ }
351
+
352
+ struct llama_sampler * llama_sampler_clone(const struct llama_sampler * smpl) {
353
+ if (smpl->iface->clone) {
354
+ return smpl->iface->clone(smpl);
355
+ }
356
+
357
+ if (smpl->ctx == nullptr) {
358
+ return llama_sampler_init(
359
+ /* .iface = */ smpl->iface,
360
+ /* .ctx = */ nullptr
361
+ );
362
+ }
363
+
364
+ GGML_ABORT("the sampler does not support cloning");
365
+ }
366
+
367
+ void llama_sampler_free(struct llama_sampler * smpl) {
368
+ if (smpl == nullptr) {
369
+ return;
370
+ }
371
+
372
+ if (smpl->iface->free) {
373
+ smpl->iface->free(smpl);
374
+ }
375
+
376
+ delete smpl;
377
+ }
378
+
379
+ llama_token llama_sampler_sample(struct llama_sampler * smpl, struct llama_context * ctx, int32_t idx) {
380
+ const auto * logits = llama_get_logits_ith(ctx, idx);
381
+
382
+ const llama_model * model = llama_get_model(ctx);
383
+ const llama_vocab * vocab = llama_model_get_vocab(model);
384
+
385
+ const int n_vocab = llama_vocab_n_tokens(vocab);
386
+
387
+ // TODO: do not allocate each time
388
+ std::vector<llama_token_data> cur;
389
+ cur.reserve(n_vocab);
390
+ for (llama_token token_id = 0; token_id < n_vocab; token_id++) {
391
+ cur.emplace_back(llama_token_data{token_id, logits[token_id], 0.0f});
392
+ }
393
+
394
+ llama_token_data_array cur_p = {
395
+ /* .data = */ cur.data(),
396
+ /* .size = */ cur.size(),
397
+ /* .selected = */ -1,
398
+ /* .sorted = */ false,
399
+ };
400
+
401
+ llama_sampler_apply(smpl, &cur_p);
402
+
403
+ GGML_ASSERT(cur_p.selected >= 0 && cur_p.selected < (int32_t) cur_p.size);
404
+
405
+ auto token = cur_p.data[cur_p.selected].id;
406
+
407
+ llama_sampler_accept(smpl, token);
408
+
409
+ return token;
410
+ }
411
+
412
+ // sampler chain
413
+
414
+ static const char * llama_sampler_chain_name(const struct llama_sampler * /*smpl*/) {
415
+ return "chain";
416
+ }
417
+
418
+ static void llama_sampler_chain_accept(struct llama_sampler * smpl, llama_token token) {
419
+ auto * chain = (llama_sampler_chain *) smpl->ctx;
420
+
421
+ time_meas tm(chain->t_sample_us, chain->params.no_perf);
422
+
423
+ for (auto * smpl : chain->samplers) {
424
+ llama_sampler_accept(smpl, token);
425
+ }
426
+
427
+ chain->n_sample++;
428
+ }
429
+
430
+ static void llama_sampler_chain_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
431
+ auto * chain = (llama_sampler_chain *) smpl->ctx;
432
+
433
+ time_meas tm(chain->t_sample_us, chain->params.no_perf);
434
+
435
+ for (auto * smpl : chain->samplers) {
436
+ llama_sampler_apply(smpl, cur_p);
437
+ }
438
+ }
439
+
440
+ static void llama_sampler_chain_reset(struct llama_sampler * smpl) {
441
+ auto * chain = (llama_sampler_chain *) smpl->ctx;
442
+
443
+ for (auto * smpl : chain->samplers) {
444
+ llama_sampler_reset(smpl);
445
+ }
446
+
447
+ chain->t_sample_us = 0;
448
+ chain->n_sample = 0;
449
+ }
450
+
451
+ static struct llama_sampler * llama_sampler_chain_clone(const struct llama_sampler * smpl) {
452
+ const auto * chain_src = (const llama_sampler_chain *) smpl->ctx;
453
+
454
+ auto * result = llama_sampler_chain_init(chain_src->params);
455
+
456
+ for (auto * smpl : chain_src->samplers) {
457
+ llama_sampler_chain_add(result, llama_sampler_clone(smpl));
458
+ }
459
+
460
+ return result;
461
+ }
462
+
463
+ static void llama_sampler_chain_free(struct llama_sampler * smpl) {
464
+ auto * chain = (llama_sampler_chain *) smpl->ctx;
465
+
466
+ for (auto * smpl : chain->samplers) {
467
+ llama_sampler_free(smpl);
468
+ }
469
+
470
+ delete chain;
471
+ }
472
+
473
+ static struct llama_sampler_i llama_sampler_chain_i = {
474
+ /* .name = */ llama_sampler_chain_name,
475
+ /* .accept = */ llama_sampler_chain_accept,
476
+ /* .apply = */ llama_sampler_chain_apply,
477
+ /* .reset = */ llama_sampler_chain_reset,
478
+ /* .clone = */ llama_sampler_chain_clone,
479
+ /* .free = */ llama_sampler_chain_free,
480
+ };
481
+
482
+ struct llama_sampler * llama_sampler_chain_init(struct llama_sampler_chain_params params) {
483
+ return llama_sampler_init(
484
+ /* .iface = */ &llama_sampler_chain_i,
485
+ /* .ctx = */ new llama_sampler_chain {
486
+ /* .params = */ params,
487
+ /* .samplers = */ {},
488
+ /* .t_sample_us = */ 0,
489
+ /* .n_sample = */ 0,
490
+ }
491
+ );
492
+ }
493
+
494
+ void llama_sampler_chain_add(struct llama_sampler * chain, struct llama_sampler * smpl) {
495
+ auto * p = (llama_sampler_chain *) chain->ctx;
496
+ p->samplers.push_back(smpl);
497
+ }
498
+
499
+ struct llama_sampler * llama_sampler_chain_get(const struct llama_sampler * chain, int32_t i) {
500
+ const auto * p = (const llama_sampler_chain *) chain->ctx;
501
+
502
+ if (i < 0 || (size_t) i >= p->samplers.size()) {
503
+ return nullptr;
504
+ }
505
+
506
+ return p->samplers[i];
507
+ }
508
+
509
+ struct llama_sampler * llama_sampler_chain_remove(struct llama_sampler * chain, int32_t i) {
510
+ auto * p = (llama_sampler_chain *) chain->ctx;
511
+
512
+ if (i < 0 || (size_t) i >= p->samplers.size()) {
513
+ return nullptr;
514
+ }
515
+
516
+ auto * result = p->samplers[i];
517
+ p->samplers.erase(p->samplers.begin() + i);
518
+
519
+ return result;
520
+ }
521
+
522
+ int llama_sampler_chain_n(const struct llama_sampler * chain) {
523
+ const auto * p = (const llama_sampler_chain *) chain->ctx;
524
+
525
+ return p->samplers.size();
526
+ }
527
+
528
+ //
529
+ // samplers
530
+ //
531
+
532
+ // greedy
533
+
534
+ static const char * llama_sampler_greedy_name(const struct llama_sampler * /*smpl*/) {
535
+ return "greedy";
536
+ }
537
+
538
+ static void llama_sampler_greedy_apply(struct llama_sampler * /*smpl*/, llama_token_data_array * cur_p) {
539
+ cur_p->selected = 0;
540
+ for (size_t i = 1; i < cur_p->size; ++i) {
541
+ if (cur_p->data[i].logit > cur_p->data[cur_p->selected].logit) {
542
+ cur_p->selected = i;
543
+ }
544
+ }
545
+ }
546
+
547
+ static struct llama_sampler_i llama_sampler_greedy_i = {
548
+ /* .name = */ llama_sampler_greedy_name,
549
+ /* .accept = */ nullptr,
550
+ /* .apply = */ llama_sampler_greedy_apply,
551
+ /* .reset = */ nullptr,
552
+ /* .clone = */ nullptr,
553
+ /* .free = */ nullptr,
554
+ };
555
+
556
+ struct llama_sampler * llama_sampler_init_greedy() {
557
+ return llama_sampler_init(
558
+ /* .iface = */ &llama_sampler_greedy_i,
559
+ /* .ctx = */ nullptr
560
+ );
561
+ }
562
+
563
+ // dist
564
+
565
+ struct llama_sampler_dist {
566
+ const uint32_t seed;
567
+ uint32_t seed_cur;
568
+
569
+ std::mt19937 rng;
570
+ };
571
+
572
+ static const char * llama_sampler_dist_name(const struct llama_sampler * /*smpl*/) {
573
+ return "dist";
574
+ }
575
+
576
+ static void llama_sampler_dist_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
577
+ auto * ctx = (llama_sampler_dist *) smpl->ctx;
578
+
579
+ llama_sampler_softmax_impl(cur_p);
580
+
581
+ cur_p->selected = llama_sample_dist(cur_p, ctx->rng);
582
+ }
583
+
584
+ static struct llama_sampler * llama_sampler_dist_clone(const struct llama_sampler * smpl) {
585
+ const auto * ctx = (const llama_sampler_dist *) smpl->ctx;
586
+ auto * result = llama_sampler_init_dist(ctx->seed);
587
+
588
+ // copy the state
589
+ {
590
+ auto * result_ctx = (llama_sampler_dist *) result->ctx;
591
+
592
+ result_ctx->rng = ctx->rng;
593
+ }
594
+
595
+ return result;
596
+ }
597
+
598
+ static void llama_sampler_dist_reset(struct llama_sampler * smpl) {
599
+ auto * ctx = (llama_sampler_dist *) smpl->ctx;
600
+ ctx->seed_cur = get_rng_seed(ctx->seed);
601
+ ctx->rng.seed(ctx->seed_cur);
602
+ }
603
+
604
+ static void llama_sampler_dist_free(struct llama_sampler * smpl) {
605
+ delete (llama_sampler_dist *) smpl->ctx;
606
+ }
607
+
608
+ static struct llama_sampler_i llama_sampler_dist_i = {
609
+ /* .name = */ llama_sampler_dist_name,
610
+ /* .accept = */ nullptr,
611
+ /* .apply = */ llama_sampler_dist_apply,
612
+ /* .reset = */ llama_sampler_dist_reset,
613
+ /* .clone = */ llama_sampler_dist_clone,
614
+ /* .free = */ llama_sampler_dist_free,
615
+ };
616
+
617
+ struct llama_sampler * llama_sampler_init_dist(uint32_t seed) {
618
+ auto seed_cur = get_rng_seed(seed);
619
+ return llama_sampler_init(
620
+ /* .iface = */ &llama_sampler_dist_i,
621
+ /* .ctx = */ new llama_sampler_dist {
622
+ /* .seed = */ seed,
623
+ /* .seed_cur = */ seed_cur,
624
+ /* .rng = */ std::mt19937(seed_cur),
625
+ }
626
+ );
627
+ }
628
+
629
+ // softmax
630
+
631
+ static const char * llama_sampler_softmax_name(const struct llama_sampler * /*smpl*/) {
632
+ return "softmax";
633
+ }
634
+
635
+ static void llama_sampler_softmax_apply(struct llama_sampler * /*smpl*/, llama_token_data_array * cur_p) {
636
+ llama_sampler_softmax_impl(cur_p);
637
+ }
638
+
639
+ static struct llama_sampler_i llama_sampler_softmax_i = {
640
+ /* .name = */ llama_sampler_softmax_name,
641
+ /* .accept = */ nullptr,
642
+ /* .apply = */ llama_sampler_softmax_apply,
643
+ /* .reset = */ nullptr,
644
+ /* .clone = */ nullptr,
645
+ /* .free = */ nullptr,
646
+ };
647
+
648
+ struct llama_sampler * llama_sampler_init_softmax() {
649
+ return llama_sampler_init(
650
+ /* .iface = */ &llama_sampler_softmax_i,
651
+ /* .ctx = */ nullptr
652
+ );
653
+ }
654
+
655
+ // top-k
656
+
657
+ struct llama_sampler_top_k {
658
+ const int32_t k;
659
+ };
660
+
661
+ static const char * llama_sampler_top_k_name(const struct llama_sampler * /*smpl*/) {
662
+ return "top-k";
663
+ }
664
+
665
+ static void llama_sampler_top_k_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
666
+ const auto * ctx = (llama_sampler_top_k *) smpl->ctx;
667
+ llama_sampler_top_k_impl(cur_p, ctx->k);
668
+ }
669
+
670
+ static struct llama_sampler * llama_sampler_top_k_clone(const struct llama_sampler * smpl) {
671
+ const auto * ctx = (const llama_sampler_top_k *) smpl->ctx;
672
+ return llama_sampler_init_top_k(ctx->k);
673
+ }
674
+
675
+ static void llama_sampler_top_k_free(struct llama_sampler * smpl) {
676
+ delete (llama_sampler_top_k *) smpl->ctx;
677
+ }
678
+
679
+ static struct llama_sampler_i llama_sampler_top_k_i = {
680
+ /* .name = */ llama_sampler_top_k_name,
681
+ /* .accept = */ nullptr,
682
+ /* .apply = */ llama_sampler_top_k_apply,
683
+ /* .reset = */ nullptr,
684
+ /* .clone = */ llama_sampler_top_k_clone,
685
+ /* .free = */ llama_sampler_top_k_free,
686
+ };
687
+
688
+ struct llama_sampler * llama_sampler_init_top_k(int32_t k) {
689
+ return llama_sampler_init(
690
+ /* .iface = */ &llama_sampler_top_k_i,
691
+ /* .ctx = */ new llama_sampler_top_k {
692
+ /* .k = */ k,
693
+ }
694
+ );
695
+ }
696
+
697
+ // top-p
698
+
699
+ struct llama_sampler_top_p {
700
+ const float p;
701
+ const size_t min_keep;
702
+ };
703
+
704
+ static const char * llama_sampler_top_p_name(const struct llama_sampler * /*smpl*/) {
705
+ return "top-p";
706
+ }
707
+
708
+ static void llama_sampler_top_p_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
709
+ const auto * ctx = (llama_sampler_top_p *) smpl->ctx;
710
+
711
+ if (ctx->p >= 1.0f) {
712
+ return;
713
+ }
714
+
715
+ llama_sampler_softmax_impl(cur_p);
716
+
717
+ // Compute the cumulative probabilities
718
+ float cum_sum = 0.0f;
719
+ size_t last_idx = cur_p->size;
720
+
721
+ for (size_t i = 0; i < cur_p->size; ++i) {
722
+ cum_sum += cur_p->data[i].p;
723
+
724
+ // Check if the running sum is at least p or if we have kept at least min_keep tokens
725
+ // we set the last index to i+1 to indicate that the current iterate should be included in the set
726
+ if (cum_sum >= ctx->p && i + 1 >= ctx->min_keep) {
727
+ last_idx = i + 1;
728
+ break;
729
+ }
730
+ }
731
+
732
+ // Resize the output vector to keep only the top-p tokens
733
+ cur_p->size = last_idx;
734
+ }
735
+
736
+ static struct llama_sampler * llama_sampler_top_p_clone(const struct llama_sampler * smpl) {
737
+ const auto * ctx = (const llama_sampler_top_p *) smpl->ctx;
738
+ return llama_sampler_init_top_p(ctx->p, ctx->min_keep);
739
+ }
740
+
741
+ static void llama_sampler_top_p_free(struct llama_sampler * smpl) {
742
+ delete (llama_sampler_top_p *) smpl->ctx;
743
+ }
744
+
745
+ static struct llama_sampler_i llama_sampler_top_p_i = {
746
+ /* .name = */ llama_sampler_top_p_name,
747
+ /* .accept = */ nullptr,
748
+ /* .apply = */ llama_sampler_top_p_apply,
749
+ /* .reset = */ nullptr,
750
+ /* .clone = */ llama_sampler_top_p_clone,
751
+ /* .free = */ llama_sampler_top_p_free,
752
+ };
753
+
754
+ struct llama_sampler * llama_sampler_init_top_p(float p, size_t min_keep) {
755
+ return llama_sampler_init(
756
+ /* .iface = */ &llama_sampler_top_p_i,
757
+ /* .ctx = */ new llama_sampler_top_p {
758
+ /* .p = */ p,
759
+ /* .min_keep = */ min_keep,
760
+ }
761
+ );
762
+ }
763
+
764
+ // min-p
765
+
766
+ struct llama_sampler_min_p {
767
+ const float p;
768
+ const size_t min_keep;
769
+ };
770
+
771
+ static const char * llama_sampler_min_p_name(const struct llama_sampler * /*smpl*/) {
772
+ return "min-p";
773
+ }
774
+
775
+ static void llama_sampler_min_p_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
776
+ const auto * ctx = (llama_sampler_min_p *) smpl->ctx;
777
+
778
+ if (ctx->p <= 0.0f || !cur_p->size) {
779
+ return;
780
+ }
781
+
782
+ bool min_p_applied = false;
783
+
784
+ // if the cur_p aren't sorted, try the unsorted implementation first
785
+ if (!cur_p->sorted) {
786
+ std::vector<llama_token_data> filtered_tokens;
787
+
788
+ float max_logit = -FLT_MAX;
789
+ for (size_t i = 0; i < cur_p->size; ++i) {
790
+ max_logit = std::max(max_logit, cur_p->data[i].logit);
791
+ }
792
+ const float min_logit = max_logit + logf(ctx->p); // min logit for p_i >= p * p_max
793
+
794
+ for (size_t i = 0; i < cur_p->size; ++i) {
795
+ if (cur_p->data[i].logit >= min_logit) {
796
+ filtered_tokens.push_back(cur_p->data[i]);
797
+ }
798
+ }
799
+
800
+ // if we have enough values the operation was a success
801
+ if (!filtered_tokens.empty() && filtered_tokens.size() >= ctx->min_keep) {
802
+ memcpy(cur_p->data, filtered_tokens.data(), filtered_tokens.size()*sizeof(llama_token_data));
803
+ cur_p->size = filtered_tokens.size();
804
+ min_p_applied = true;
805
+ }
806
+ }
807
+
808
+ // if the cur_p are sorted or the unsorted implementation failed, use this implementation
809
+ if (!min_p_applied) {
810
+ // Sort the logits in descending order
811
+ if (!cur_p->sorted) {
812
+ std::sort(cur_p->data, cur_p->data + cur_p->size, [](const llama_token_data & a, const llama_token_data & b) {
813
+ return a.logit > b.logit;
814
+ });
815
+ cur_p->sorted = true;
816
+ }
817
+
818
+ const float min_logit = cur_p->data[0].logit + logf(ctx->p); // min logit for p_i >= p * p_max
819
+ size_t i = 1; // first token always matches
820
+
821
+ for (; i < cur_p->size; ++i) {
822
+ if (cur_p->data[i].logit < min_logit && i >= ctx->min_keep) {
823
+ break; // prob too small
824
+ }
825
+ }
826
+
827
+ // Resize the output vector to keep only the matching tokens
828
+ cur_p->size = i;
829
+ }
830
+ }
831
+
832
+ static struct llama_sampler * llama_sampler_min_p_clone(const struct llama_sampler * smpl) {
833
+ const auto * ctx = (const llama_sampler_min_p *) smpl->ctx;
834
+ return llama_sampler_init_min_p(ctx->p, ctx->min_keep);
835
+ }
836
+
837
+ static void llama_sampler_min_p_free(struct llama_sampler * smpl) {
838
+ delete (llama_sampler_min_p *) smpl->ctx;
839
+ }
840
+
841
+ static struct llama_sampler_i llama_sampler_min_p_i = {
842
+ /* .name = */ llama_sampler_min_p_name,
843
+ /* .accept = */ nullptr,
844
+ /* .apply = */ llama_sampler_min_p_apply,
845
+ /* .reset = */ nullptr,
846
+ /* .clone = */ llama_sampler_min_p_clone,
847
+ /* .free = */ llama_sampler_min_p_free,
848
+ };
849
+
850
+ struct llama_sampler * llama_sampler_init_min_p(float p, size_t min_keep) {
851
+ return llama_sampler_init(
852
+ /* .iface = */ &llama_sampler_min_p_i,
853
+ /* .ctx = */ new llama_sampler_min_p {
854
+ /* .p = */ p,
855
+ /* .min_keep = */ min_keep,
856
+ }
857
+ );
858
+ }
859
+
860
+ // typical
861
+
862
+ struct llama_sampler_typical {
863
+ const float p;
864
+ const size_t min_keep;
865
+ };
866
+
867
+ static const char * llama_sampler_typical_name(const struct llama_sampler * /*smpl*/) {
868
+ return "typical";
869
+ }
870
+
871
+ static void llama_sampler_typical_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
872
+ const auto * ctx = (llama_sampler_typical *) smpl->ctx;
873
+
874
+ // Reference implementation:
875
+ // https://github.com/huggingface/transformers/compare/main...cimeister:typical-sampling:typical-pr
876
+ if (ctx->p >= 1.0f) {
877
+ return;
878
+ }
879
+
880
+ // Compute the softmax of logits and calculate entropy
881
+ llama_sampler_softmax_impl(cur_p);
882
+
883
+ float entropy = 0.0f;
884
+ for (size_t i = 0; i < cur_p->size; ++i) {
885
+ entropy += -cur_p->data[i].p * logf(cur_p->data[i].p);
886
+ }
887
+
888
+ // Compute the absolute difference between negative log probability and entropy for each candidate
889
+ std::vector<float> shifted_scores;
890
+ for (size_t i = 0; i < cur_p->size; ++i) {
891
+ float shifted_score = fabsf(-logf(cur_p->data[i].p) - entropy);
892
+ shifted_scores.push_back(shifted_score);
893
+ }
894
+
895
+ // Sort tokens based on the shifted_scores and their corresponding indices
896
+ std::vector<size_t> indices(cur_p->size);
897
+ std::iota(indices.begin(), indices.end(), 0);
898
+
899
+ std::sort(indices.begin(), indices.end(), [&](size_t a, size_t b) {
900
+ return shifted_scores[a] < shifted_scores[b];
901
+ });
902
+
903
+ // Compute the cumulative probabilities
904
+ float cum_sum = 0.0f;
905
+ size_t last_idx = indices.size();
906
+
907
+ for (size_t i = 0; i < indices.size(); ++i) {
908
+ size_t idx = indices[i];
909
+ cum_sum += cur_p->data[idx].p;
910
+
911
+ // Check if the running sum is greater than typical or if we have kept at least min_keep tokens
912
+ if (cum_sum > ctx->p && (ctx->min_keep == 0 || i >= ctx->min_keep - 1)) {
913
+ last_idx = i + 1;
914
+ break;
915
+ }
916
+ }
917
+
918
+ // Resize the output vector to keep only the locally typical tokens
919
+ std::vector<llama_token_data> cur_p_new;
920
+ for (size_t i = 0; i < last_idx; ++i) {
921
+ size_t idx = indices[i];
922
+ cur_p_new.push_back(cur_p->data[idx]);
923
+ }
924
+
925
+ // Replace the data in cur_p with the cur_p_new data
926
+ std::copy(cur_p_new.begin(), cur_p_new.end(), cur_p->data);
927
+ cur_p->size = cur_p_new.size();
928
+ cur_p->sorted = false;
929
+ }
930
+
931
+ static struct llama_sampler * llama_sampler_typical_clone(const struct llama_sampler * smpl) {
932
+ const auto * ctx = (const llama_sampler_typical *) smpl->ctx;
933
+ return llama_sampler_init_typical(ctx->p, ctx->min_keep);
934
+ }
935
+
936
+ static void llama_sampler_typical_free(struct llama_sampler * smpl) {
937
+ delete (llama_sampler_typical *) smpl->ctx;
938
+ }
939
+
940
+ static struct llama_sampler_i llama_sampler_typical_i = {
941
+ /* .name = */ llama_sampler_typical_name,
942
+ /* .accept = */ nullptr,
943
+ /* .apply = */ llama_sampler_typical_apply,
944
+ /* .reset = */ nullptr,
945
+ /* .clone = */ llama_sampler_typical_clone,
946
+ /* .free = */ llama_sampler_typical_free,
947
+ };
948
+
949
+ struct llama_sampler * llama_sampler_init_typical(float p, size_t min_keep) {
950
+ return llama_sampler_init(
951
+ /* .iface = */ &llama_sampler_typical_i,
952
+ /* .ctx = */ new llama_sampler_typical {
953
+ /* .p = */ p,
954
+ /* .min_keep = */ min_keep,
955
+ }
956
+ );
957
+ }
958
+
959
+ // temp
960
+
961
+ struct llama_sampler_temp {
962
+ const float temp;
963
+ };
964
+
965
+ static const char * llama_sampler_temp_name(const struct llama_sampler * /*smpl*/) {
966
+ return "temp";
967
+ }
968
+
969
+ static void llama_sampler_temp_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
970
+ const auto * ctx = (llama_sampler_temp *) smpl->ctx;
971
+
972
+ llama_sampler_temp_impl(cur_p, ctx->temp);
973
+ }
974
+
975
+ static struct llama_sampler * llama_sampler_temp_clone(const struct llama_sampler * smpl) {
976
+ const auto * ctx = (const llama_sampler_temp *) smpl->ctx;
977
+ return llama_sampler_init_temp(ctx->temp);
978
+ }
979
+
980
+ static void llama_sampler_temp_free(struct llama_sampler * smpl) {
981
+ delete (llama_sampler_temp *) smpl->ctx;
982
+ }
983
+
984
+ static struct llama_sampler_i llama_sampler_temp_i = {
985
+ /* .name = */ llama_sampler_temp_name,
986
+ /* .accept = */ nullptr,
987
+ /* .apply = */ llama_sampler_temp_apply,
988
+ /* .reset = */ nullptr,
989
+ /* .clone = */ llama_sampler_temp_clone,
990
+ /* .free = */ llama_sampler_temp_free,
991
+ };
992
+
993
+ struct llama_sampler * llama_sampler_init_temp(float temp) {
994
+ return llama_sampler_init(
995
+ /* .iface = */ &llama_sampler_temp_i,
996
+ /* .ctx = */ new llama_sampler_temp {
997
+ /*.temp = */ temp,
998
+ }
999
+ );
1000
+ }
1001
+
1002
+ // temp-ext
1003
+
1004
+ struct llama_sampler_temp_ext {
1005
+ const float temp;
1006
+ const float delta;
1007
+ const float exponent;
1008
+ };
1009
+
1010
+ static const char * llama_sampler_temp_ext_name(const struct llama_sampler * /*smpl*/) {
1011
+ return "temp-ext";
1012
+ }
1013
+
1014
+ static void llama_sampler_temp_ext_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
1015
+ const auto * ctx = (llama_sampler_temp_ext *) smpl->ctx;
1016
+ if (ctx->delta > 0) {
1017
+ const float min_temp = std::max(0.0f, ctx->temp - ctx->delta);
1018
+ const float max_temp = ctx->temp + ctx->delta;
1019
+
1020
+ float exponent_val = ctx->exponent;
1021
+
1022
+ // no need to do anything if there is only one (or zero) candidates
1023
+ if (cur_p->size <= 1) {
1024
+ return;
1025
+ }
1026
+
1027
+ // Calculate maximum possible entropy
1028
+ float max_entropy = -logf(1.0f / cur_p->size);
1029
+
1030
+ llama_sampler_softmax_impl(cur_p);
1031
+
1032
+ // Calculate entropy of the softmax probabilities
1033
+ float entropy = 0.0f;
1034
+ for (size_t i = 0; i < cur_p->size; ++i) {
1035
+ float prob = cur_p->data[i].p;
1036
+ if (prob > 0.0f) { // Ensure no log(0)
1037
+ entropy -= prob * logf(prob);
1038
+ }
1039
+ }
1040
+
1041
+ // Normalize the entropy (max_entropy cannot be 0 here because we checked cur_p->size != 1 above)
1042
+ float normalized_entropy = entropy / max_entropy;
1043
+
1044
+ // Map the normalized entropy to the desired temperature range using the power function
1045
+ float dyn_temp = min_temp + (max_temp - min_temp) * powf(normalized_entropy, exponent_val);
1046
+
1047
+ #ifdef DEBUG
1048
+ LLAMA_LOG_INFO("Your text maxtemp value is: %f\n", max_temp);
1049
+ LLAMA_LOG_INFO("Entropy: %f\n", entropy);
1050
+ LLAMA_LOG_INFO("Max Possible Entropy: %f\n", max_entropy);
1051
+ LLAMA_LOG_INFO("Normalized Entropy: %f\n", normalized_entropy);
1052
+ LLAMA_LOG_INFO("Exponent: %f\n", exponent_val);
1053
+ LLAMA_LOG_INFO("Dynamic Temperature (dyn_temp): %f\n", dyn_temp);
1054
+ #endif
1055
+
1056
+ // Apply the dynamically calculated temperature scaling
1057
+ llama_sampler_temp_impl(cur_p, dyn_temp);
1058
+
1059
+ // Re-compute softmax probabilities after scaling logits with dynamic temperature
1060
+ const double max_l_double = cur_p->data[0].logit;
1061
+
1062
+ double cum_sum_double = 0.0;
1063
+ for (size_t i = 0; i < cur_p->size; ++i) {
1064
+ double p = exp(cur_p->data[i].logit - max_l_double);
1065
+ cur_p->data[i].p = p; // Store the scaled probability
1066
+ cum_sum_double += p;
1067
+ }
1068
+
1069
+ for (size_t i = 0; i < cur_p->size; ++i) {
1070
+ cur_p->data[i].p /= cum_sum_double; // Re-normalize the probabilities
1071
+ }
1072
+
1073
+ #ifdef DEBUG
1074
+ // Print the updated top 25 probabilities after temperature scaling
1075
+ LLAMA_LOG_INFO("\nUpdated Top 25 Probabilities After Dynamic Temperature Scaling (in percentages):\n");
1076
+ for (size_t i = 0; i < 25 && i < cur_p->size; ++i) {
1077
+ LLAMA_LOG_INFO("Token %zu: %f%%\n", i + 1, cur_p->data[i].p * 100.0f);
1078
+ }
1079
+ #endif
1080
+ } else {
1081
+ llama_sampler_temp_impl(cur_p, ctx->temp);
1082
+ }
1083
+ }
1084
+
1085
+ static struct llama_sampler * llama_sampler_temp_ext_clone(const struct llama_sampler * smpl) {
1086
+ const auto * ctx = (const llama_sampler_temp_ext *) smpl->ctx;
1087
+ return llama_sampler_init_temp_ext(ctx->temp, ctx->delta, ctx->exponent);
1088
+ }
1089
+
1090
+ static void llama_sampler_temp_ext_free(struct llama_sampler * smpl) {
1091
+ delete (llama_sampler_temp_ext *) smpl->ctx;
1092
+ }
1093
+
1094
+ static struct llama_sampler_i llama_sampler_temp_ext_i = {
1095
+ /* .name = */ llama_sampler_temp_ext_name,
1096
+ /* .accept = */ nullptr,
1097
+ /* .apply = */ llama_sampler_temp_ext_apply,
1098
+ /* .reset = */ nullptr,
1099
+ /* .clone = */ llama_sampler_temp_ext_clone,
1100
+ /* .free = */ llama_sampler_temp_ext_free,
1101
+ };
1102
+
1103
+ struct llama_sampler * llama_sampler_init_temp_ext(float temp, float delta, float exponent) {
1104
+ return llama_sampler_init(
1105
+ /* .iface = */ &llama_sampler_temp_ext_i,
1106
+ /* .ctx = */ new llama_sampler_temp_ext {
1107
+ /* .temp = */ temp,
1108
+ /* .delta = */ delta,
1109
+ /* .exponent = */ exponent,
1110
+ }
1111
+ );
1112
+ }
1113
+
1114
+ // xtc
1115
+
1116
+ struct llama_sampler_xtc {
1117
+ const float probability;
1118
+ const float threshold;
1119
+ const size_t min_keep;
1120
+
1121
+ const uint32_t seed;
1122
+ uint32_t seed_cur;
1123
+
1124
+ std::mt19937 rng;
1125
+ };
1126
+
1127
+ static const char * llama_sampler_xtc_name(const struct llama_sampler * /*smpl*/) {
1128
+ return "xtc";
1129
+ }
1130
+
1131
+ static void llama_sample_xtc_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
1132
+ auto * ctx = (llama_sampler_xtc *) smpl->ctx;
1133
+
1134
+ if (ctx->probability <= 0.0f
1135
+ || ctx->threshold > 0.5f
1136
+ || cur_p->size < 2) {
1137
+ return;
1138
+ }
1139
+
1140
+ std::uniform_real_distribution<float> distribution(0.0f, 1.0f);
1141
+ float chance = distribution(ctx->rng);
1142
+ if (chance > ctx->probability) return;
1143
+
1144
+ // in case it's not sorted/recalculated yet
1145
+ llama_sampler_softmax_impl(cur_p);
1146
+
1147
+ int pos_last = 0;
1148
+
1149
+ for (size_t i = 0; i < cur_p->size; ++i) {
1150
+ if (cur_p->data[i].p >= ctx->threshold) {
1151
+ pos_last = i;
1152
+ } else break;
1153
+ }
1154
+
1155
+ if (cur_p->size - pos_last >= ctx->min_keep && pos_last > 0) {
1156
+ cur_p->data += pos_last;
1157
+ cur_p->size -= pos_last;
1158
+ }
1159
+ }
1160
+
1161
+ static struct llama_sampler * llama_sampler_xtc_clone(const struct llama_sampler * smpl) {
1162
+ const auto * ctx = (const llama_sampler_xtc *) smpl->ctx;
1163
+ auto * result = llama_sampler_init_xtc(ctx->probability, ctx->threshold, ctx->min_keep, ctx->seed);
1164
+
1165
+ // copy the state
1166
+ {
1167
+ auto * result_ctx = (llama_sampler_xtc *) result->ctx;
1168
+
1169
+ result_ctx->rng = ctx->rng;
1170
+ }
1171
+
1172
+ return result;
1173
+ }
1174
+
1175
+ static void llama_sampler_xtc_free(struct llama_sampler * smpl) {
1176
+ delete (llama_sampler_xtc *) smpl->ctx;
1177
+ }
1178
+
1179
+ static void llama_sampler_xtc_reset(struct llama_sampler * smpl) {
1180
+ auto * ctx = (llama_sampler_xtc *) smpl->ctx;
1181
+ ctx->seed_cur = get_rng_seed(ctx->seed);
1182
+ ctx->rng.seed(ctx->seed_cur);
1183
+ }
1184
+
1185
+ static struct llama_sampler_i llama_sampler_xtc_i = {
1186
+ /* .name = */ llama_sampler_xtc_name,
1187
+ /* .accept = */ nullptr,
1188
+ /* .apply = */ llama_sample_xtc_apply,
1189
+ /* .reset = */ llama_sampler_xtc_reset,
1190
+ /* .clone = */ llama_sampler_xtc_clone,
1191
+ /* .free = */ llama_sampler_xtc_free,
1192
+ };
1193
+
1194
+ struct llama_sampler * llama_sampler_init_xtc(float p, float t, size_t min_keep, uint32_t seed) {
1195
+ auto seed_cur = get_rng_seed(seed);
1196
+ return llama_sampler_init(
1197
+ /* .iface = */ &llama_sampler_xtc_i,
1198
+ /* .ctx = */ new llama_sampler_xtc {
1199
+ /* .probability = */ p,
1200
+ /* .threshold = */ t,
1201
+ /* .min_keep = */ min_keep,
1202
+ /* .seed = */ seed,
1203
+ /* .seed_cur = */ seed_cur,
1204
+ /* .rng = */ std::mt19937(seed_cur),
1205
+ }
1206
+ );
1207
+ }
1208
+
1209
+ // mirostat
1210
+
1211
+ struct llama_sampler_mirostat {
1212
+ const int32_t n_vocab;
1213
+
1214
+ const uint32_t seed;
1215
+ uint32_t seed_cur;
1216
+
1217
+ const float tau;
1218
+ const float eta;
1219
+
1220
+ const int32_t m;
1221
+
1222
+ float mu;
1223
+
1224
+ std::mt19937 rng;
1225
+ };
1226
+
1227
+ static const char * llama_sampler_mirostat_name(const struct llama_sampler * /*smpl*/) {
1228
+ return "mirostat";
1229
+ }
1230
+
1231
+ static void llama_sampler_mirostat_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
1232
+ auto * ctx = (llama_sampler_mirostat *) smpl->ctx;
1233
+
1234
+ llama_sampler_softmax_impl(cur_p);
1235
+
1236
+ // Estimate s_hat using the most probable m tokens
1237
+ float s_hat = 0.0;
1238
+ float sum_ti_bi = 0.0;
1239
+ float sum_ti_sq = 0.0;
1240
+ for (size_t i = 0; i < size_t(ctx->m - 1) && i < cur_p->size - 1; ++i) {
1241
+ float t_i = logf(float(i + 2) / float(i + 1));
1242
+ float b_i = logf(cur_p->data[i].p / cur_p->data[i + 1].p);
1243
+ sum_ti_bi += t_i * b_i;
1244
+ sum_ti_sq += t_i * t_i;
1245
+ }
1246
+ s_hat = sum_ti_bi / sum_ti_sq;
1247
+
1248
+ // Compute k from the estimated s_hat and target surprise value
1249
+ float epsilon_hat = s_hat - 1;
1250
+ float k = powf((epsilon_hat * powf(2, ctx->mu)) / (1 - powf(ctx->n_vocab, -epsilon_hat)), 1 / s_hat);
1251
+
1252
+ llama_sampler_top_k_impl(cur_p, std::max(int(k), 1));
1253
+ llama_sampler_softmax_impl(cur_p);
1254
+
1255
+ const int idx = llama_sample_dist(cur_p, ctx->rng);
1256
+
1257
+ cur_p->selected = idx;
1258
+
1259
+ float observed_surprise = -log2f(cur_p->data[idx].p);
1260
+ float e = observed_surprise - ctx->tau;
1261
+
1262
+ // Update mu using the learning rate and error
1263
+ ctx->mu = ctx->mu - ctx->eta * e;
1264
+ }
1265
+
1266
+ static struct llama_sampler * llama_sampler_mirostat_clone(const struct llama_sampler * smpl) {
1267
+ const auto * ctx = (const llama_sampler_mirostat *) smpl->ctx;
1268
+ auto * result = llama_sampler_init_mirostat(ctx->n_vocab, ctx->seed, ctx->tau, ctx->eta, ctx->m);
1269
+
1270
+ // copy the state
1271
+ {
1272
+ auto * result_ctx = (llama_sampler_mirostat *) smpl->ctx;
1273
+
1274
+ result_ctx->mu = ctx->mu;
1275
+ result_ctx->rng = ctx->rng;
1276
+ }
1277
+
1278
+ return result;
1279
+ }
1280
+
1281
+ static void llama_sampler_mirostat_reset(struct llama_sampler * smpl) {
1282
+ auto * ctx = (llama_sampler_mirostat *) smpl->ctx;
1283
+ ctx->mu = 2.0f*ctx->tau;
1284
+ ctx->seed_cur = get_rng_seed(ctx->seed);
1285
+ ctx->rng.seed(ctx->seed_cur);
1286
+ }
1287
+
1288
+ static void llama_sampler_mirostat_free(struct llama_sampler * smpl) {
1289
+ delete (llama_sampler_mirostat *) smpl->ctx;
1290
+ }
1291
+
1292
+ static struct llama_sampler_i llama_sampler_mirostat_i = {
1293
+ /* .name = */ llama_sampler_mirostat_name,
1294
+ /* .accept = */ nullptr,
1295
+ /* .apply = */ llama_sampler_mirostat_apply,
1296
+ /* .reset = */ llama_sampler_mirostat_reset,
1297
+ /* .clone = */ llama_sampler_mirostat_clone,
1298
+ /* .free = */ llama_sampler_mirostat_free,
1299
+ };
1300
+
1301
+ struct llama_sampler * llama_sampler_init_mirostat(int32_t n_vocab, uint32_t seed, float tau, float eta, int32_t m) {
1302
+ auto seed_cur = get_rng_seed(seed);
1303
+ return llama_sampler_init(
1304
+ /* .iface = */ &llama_sampler_mirostat_i,
1305
+ /* .ctx = */ new llama_sampler_mirostat {
1306
+ /* .n_vocab = */ n_vocab,
1307
+ /* .seed = */ seed,
1308
+ /* .seed_cur = */ seed_cur,
1309
+ /* .tau = */ tau,
1310
+ /* .eta = */ eta,
1311
+ /* .m = */ m,
1312
+ /* .mu = */ 2.0f*tau,
1313
+ /* .rng = */ std::mt19937(seed_cur),
1314
+ }
1315
+ );
1316
+ }
1317
+
1318
+ // mirostat v2
1319
+
1320
+ struct llama_sampler_mirostat_v2 {
1321
+ const uint32_t seed;
1322
+ uint32_t seed_cur;
1323
+
1324
+ const float tau;
1325
+ const float eta;
1326
+
1327
+ float mu;
1328
+
1329
+ std::mt19937 rng;
1330
+ };
1331
+
1332
+ static const char * llama_sampler_mirostat_v2_name(const struct llama_sampler * /*smpl*/) {
1333
+ return "mirostat-v2";
1334
+ }
1335
+
1336
+ static void llama_sampler_mirostat_v2_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
1337
+ auto * ctx = (llama_sampler_mirostat_v2 *) smpl->ctx;
1338
+
1339
+ llama_sampler_softmax_impl(cur_p);
1340
+
1341
+ // Truncate the words with surprise values greater than mu
1342
+ cur_p->size = std::distance(cur_p->data, std::find_if(cur_p->data, cur_p->data + cur_p->size, [&](const llama_token_data & candidate) {
1343
+ return -log2f(candidate.p) > ctx->mu;
1344
+ }));
1345
+
1346
+ if (cur_p->size == 0) {
1347
+ cur_p->size = 1;
1348
+ }
1349
+
1350
+ // Normalize the probabilities of the remaining words
1351
+ llama_sampler_softmax_impl(cur_p);
1352
+
1353
+ const int idx = llama_sample_dist(cur_p, ctx->rng);
1354
+
1355
+ cur_p->selected = idx;
1356
+
1357
+ float observed_surprise = -log2f(cur_p->data[idx].p);
1358
+ float e = observed_surprise - ctx->tau;
1359
+
1360
+ // Update mu using the learning rate and error
1361
+ ctx->mu = ctx->mu - ctx->eta * e;
1362
+ }
1363
+
1364
+ static void llama_sampler_mirostat_v2_reset(struct llama_sampler * smpl) {
1365
+ auto * ctx = (llama_sampler_mirostat_v2 *) smpl->ctx;
1366
+ ctx->mu = 2.0f*ctx->tau;
1367
+ ctx->seed_cur = get_rng_seed(ctx->seed);
1368
+ ctx->rng.seed(ctx->seed_cur);
1369
+ }
1370
+
1371
+ static struct llama_sampler * llama_sampler_mirostat_v2_clone(const struct llama_sampler * smpl) {
1372
+ const auto * ctx = (const llama_sampler_mirostat_v2 *) smpl->ctx;
1373
+
1374
+ auto * result = llama_sampler_init_mirostat_v2(ctx->seed, ctx->tau, ctx->eta);
1375
+
1376
+ // copy the state
1377
+ {
1378
+ auto * result_ctx = (llama_sampler_mirostat_v2 *) result->ctx;
1379
+
1380
+ result_ctx->mu = ctx->mu;
1381
+ result_ctx->rng = ctx->rng;
1382
+ }
1383
+
1384
+ return result;
1385
+ }
1386
+
1387
+ static void llama_sampler_mirostat_v2_free(struct llama_sampler * smpl) {
1388
+ delete (llama_sampler_mirostat_v2 *) smpl->ctx;
1389
+ }
1390
+
1391
+ static struct llama_sampler_i llama_sampler_mirostat_v2_i = {
1392
+ /* .name = */ llama_sampler_mirostat_v2_name,
1393
+ /* .accept = */ nullptr,
1394
+ /* .apply = */ llama_sampler_mirostat_v2_apply,
1395
+ /* .reset = */ llama_sampler_mirostat_v2_reset,
1396
+ /* .clone = */ llama_sampler_mirostat_v2_clone,
1397
+ /* .free = */ llama_sampler_mirostat_v2_free,
1398
+ };
1399
+
1400
+ struct llama_sampler * llama_sampler_init_mirostat_v2(uint32_t seed, float tau, float eta) {
1401
+ auto seed_cur = get_rng_seed(seed);
1402
+ return llama_sampler_init(
1403
+ /* .iface = */ &llama_sampler_mirostat_v2_i,
1404
+ /* .ctx = */ new llama_sampler_mirostat_v2 {
1405
+ /* .seed = */ seed,
1406
+ /* .seed_cur = */ seed_cur,
1407
+ /* .tau = */ tau,
1408
+ /* .eta = */ eta,
1409
+ /* .mu = */ 2.0f*tau,
1410
+ /* .rng = */ std::mt19937(seed_cur),
1411
+ }
1412
+ );
1413
+ }
1414
+
1415
+ // grammar
1416
+
1417
+ struct llama_sampler_grammar {
1418
+ const struct llama_vocab * vocab;
1419
+
1420
+ std::string grammar_str;
1421
+ std::string grammar_root;
1422
+
1423
+ struct llama_grammar * grammar;
1424
+ };
1425
+
1426
+ static const char * llama_sampler_grammar_name(const struct llama_sampler * /*smpl*/) {
1427
+ return "grammar";
1428
+ }
1429
+
1430
+ static void llama_sampler_grammar_accept_impl(struct llama_sampler * smpl, llama_token token) {
1431
+ auto * ctx = (llama_sampler_grammar *) smpl->ctx;
1432
+ if (ctx->grammar) {
1433
+ llama_grammar_accept_impl(*ctx->grammar, token);
1434
+ }
1435
+ }
1436
+
1437
+ static void llama_sampler_grammar_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
1438
+ auto * ctx = (llama_sampler_grammar *) smpl->ctx;
1439
+ if (ctx->grammar) {
1440
+ llama_grammar_apply_impl(*ctx->grammar, cur_p);
1441
+ }
1442
+ }
1443
+
1444
+ // Fwd declare to break reset --> init_impl --> llama_sampler_grammar_i --> reset cycle.
1445
+ static struct llama_sampler * llama_sampler_init_grammar_impl(
1446
+ const struct llama_vocab * vocab,
1447
+ const char * grammar_str,
1448
+ const char * grammar_root,
1449
+ bool lazy,
1450
+ const char ** trigger_words,
1451
+ size_t num_trigger_words,
1452
+ const llama_token * trigger_tokens,
1453
+ size_t num_trigger_tokens,
1454
+ const char ** trigger_patterns,
1455
+ size_t num_trigger_patterns);
1456
+
1457
+ static void llama_sampler_grammar_reset(struct llama_sampler * smpl) {
1458
+ auto * ctx = (llama_sampler_grammar *) smpl->ctx;
1459
+ if (!ctx->grammar) {
1460
+ return;
1461
+ }
1462
+
1463
+ std::vector<const char *> trigger_patterns_c;
1464
+ trigger_patterns_c.reserve(ctx->grammar->trigger_patterns.size());
1465
+ for (auto & trigger_pattern : ctx->grammar->trigger_patterns) {
1466
+ trigger_patterns_c.push_back(trigger_pattern.pattern.c_str());
1467
+ }
1468
+
1469
+ auto * grammar_new = llama_grammar_init_impl(ctx->grammar->vocab, ctx->grammar_str.c_str(), ctx->grammar_root.c_str(),
1470
+ ctx->grammar->lazy, trigger_patterns_c.data(), trigger_patterns_c.size(),
1471
+ ctx->grammar->trigger_tokens.data(), ctx->grammar->trigger_tokens.size());
1472
+
1473
+ llama_grammar_free_impl(ctx->grammar);
1474
+ ctx->grammar = grammar_new;
1475
+ }
1476
+
1477
+ static struct llama_sampler * llama_sampler_grammar_clone(const struct llama_sampler * smpl) {
1478
+ const auto * ctx = (const llama_sampler_grammar *) smpl->ctx;
1479
+
1480
+ auto * result = llama_sampler_init_grammar_impl(ctx->vocab, nullptr, nullptr, false, nullptr, 0, nullptr, 0, nullptr, 0);
1481
+ GGML_ASSERT(result);
1482
+
1483
+ // copy the state
1484
+ {
1485
+ auto * result_ctx = (llama_sampler_grammar *) result->ctx;
1486
+
1487
+ if (ctx->grammar) {
1488
+ result_ctx->grammar_str = ctx->grammar_str;
1489
+ result_ctx->grammar_root = ctx->grammar_root;
1490
+
1491
+ result_ctx->grammar = llama_grammar_clone_impl(*ctx->grammar);
1492
+ }
1493
+ }
1494
+
1495
+ return result;
1496
+ }
1497
+
1498
+ static void llama_sampler_grammar_free(struct llama_sampler * smpl) {
1499
+ const auto * ctx = (llama_sampler_grammar *) smpl->ctx;
1500
+
1501
+ if (ctx->grammar) {
1502
+ llama_grammar_free_impl(ctx->grammar);
1503
+ }
1504
+
1505
+ delete ctx;
1506
+ }
1507
+
1508
+ static struct llama_sampler_i llama_sampler_grammar_i = {
1509
+ /* .name = */ llama_sampler_grammar_name,
1510
+ /* .accept = */ llama_sampler_grammar_accept_impl,
1511
+ /* .apply = */ llama_sampler_grammar_apply,
1512
+ /* .reset = */ llama_sampler_grammar_reset,
1513
+ /* .clone = */ llama_sampler_grammar_clone,
1514
+ /* .free = */ llama_sampler_grammar_free,
1515
+ };
1516
+
1517
+ static struct llama_sampler * llama_sampler_init_grammar_impl(
1518
+ const struct llama_vocab * vocab,
1519
+ const char * grammar_str,
1520
+ const char * grammar_root,
1521
+ bool lazy,
1522
+ const char ** trigger_words,
1523
+ size_t num_trigger_words,
1524
+ const llama_token * trigger_tokens,
1525
+ size_t num_trigger_tokens,
1526
+ const char ** trigger_patterns,
1527
+ size_t num_trigger_patterns) {
1528
+ auto * ctx = new llama_sampler_grammar;
1529
+
1530
+ if (grammar_str != nullptr && grammar_str[0] != '\0') {
1531
+ // TODO: remove trigger_words support.
1532
+ if (trigger_words != nullptr && num_trigger_words > 0) {
1533
+ GGML_ASSERT(trigger_patterns == nullptr && num_trigger_patterns == 0);
1534
+ std::string trigger_pattern("[\\s\\S]*?(");
1535
+ for (size_t i = 0; i < num_trigger_words; ++i) {
1536
+ static const std::regex special_chars("[.^$|()*+?\\[\\]{}\\\\]");
1537
+ if (i > 0) {
1538
+ trigger_pattern += "|";
1539
+ }
1540
+ trigger_pattern += std::regex_replace(trigger_words[i], special_chars, "\\$0");
1541
+ }
1542
+ trigger_pattern += ")[\\s\\S]*";
1543
+ auto trigger_pattern_c = trigger_pattern.c_str();
1544
+ trigger_patterns = &trigger_pattern_c;
1545
+ num_trigger_patterns = 1;
1546
+ }
1547
+ *ctx = {
1548
+ /* .vocab = */ vocab,
1549
+ /* .grammar_str = */ grammar_str,
1550
+ /* .grammar_root = */ grammar_root,
1551
+ /* .grammar = */ llama_grammar_init_impl(vocab, grammar_str, grammar_root, lazy, trigger_patterns, num_trigger_patterns, trigger_tokens, num_trigger_tokens),
1552
+ };
1553
+ if (!ctx->grammar) {
1554
+ delete ctx;
1555
+ return nullptr;
1556
+ }
1557
+ } else {
1558
+ *ctx = {
1559
+ /* .vocab = */ vocab,
1560
+ /* .grammar_str = */ {},
1561
+ /* .grammar_root = */ {},
1562
+ /* .grammar = */ nullptr,
1563
+ };
1564
+ }
1565
+
1566
+ return llama_sampler_init(
1567
+ /* .iface = */ &llama_sampler_grammar_i,
1568
+ /* .ctx = */ ctx
1569
+ );
1570
+ }
1571
+
1572
+ struct llama_sampler * llama_sampler_init_grammar(
1573
+ const struct llama_vocab * vocab,
1574
+ const char * grammar_str,
1575
+ const char * grammar_root) {
1576
+ return llama_sampler_init_grammar_impl(vocab, grammar_str, grammar_root, /* lazy= */ false, nullptr, 0, nullptr, 0, nullptr, 0);
1577
+ }
1578
+
1579
+ struct llama_sampler * llama_sampler_init_grammar_lazy(
1580
+ const struct llama_vocab * vocab,
1581
+ const char * grammar_str,
1582
+ const char * grammar_root,
1583
+ const char ** trigger_words,
1584
+ size_t num_trigger_words,
1585
+ const llama_token * trigger_tokens,
1586
+ size_t num_trigger_tokens) {
1587
+ return llama_sampler_init_grammar_impl(vocab, grammar_str, grammar_root, /* lazy= */ true, trigger_words, num_trigger_words, trigger_tokens, num_trigger_tokens, nullptr, 0);
1588
+ }
1589
+
1590
+ struct llama_sampler * llama_sampler_init_grammar_lazy_patterns(
1591
+ const struct llama_vocab * vocab,
1592
+ const char * grammar_str,
1593
+ const char * grammar_root,
1594
+ const char ** trigger_patterns,
1595
+ size_t num_trigger_patterns,
1596
+ const llama_token * trigger_tokens,
1597
+ size_t num_trigger_tokens) {
1598
+ return llama_sampler_init_grammar_impl(vocab, grammar_str, grammar_root, /* lazy= */ true, nullptr, 0, trigger_tokens, num_trigger_tokens, trigger_patterns, num_trigger_patterns);
1599
+ }
1600
+
1601
+ // penalties
1602
+
1603
+ struct llama_sampler_penalties {
1604
+ const int32_t penalty_last_n;
1605
+ const float penalty_repeat;
1606
+ const float penalty_freq;
1607
+ const float penalty_present;
1608
+
1609
+ ring_buffer<llama_token> prev;
1610
+
1611
+ // a frequency map to count token occurrences
1612
+ std::unordered_map<llama_token, int> token_count;
1613
+ };
1614
+
1615
+ static const char * llama_sampler_penalties_name(const struct llama_sampler * /*smpl*/) {
1616
+ return "penalties";
1617
+ }
1618
+
1619
+ static void llama_sampler_penalties_accept(struct llama_sampler * smpl, llama_token token) {
1620
+ auto * ctx = (llama_sampler_penalties *) smpl->ctx;
1621
+ if (ctx->penalty_last_n == 0) {
1622
+ return;
1623
+ }
1624
+
1625
+ ctx->token_count[token]++;
1626
+
1627
+ // if the ring buffer is full, remove the oldest token
1628
+ if (ctx->prev.size() >= (size_t) ctx->penalty_last_n) {
1629
+ const auto old = ctx->prev.front();
1630
+
1631
+ ctx->token_count[old]--;
1632
+ if (ctx->token_count[old] == 0) {
1633
+ ctx->token_count.erase(old);
1634
+ }
1635
+ }
1636
+
1637
+ ctx->prev.push_back(token);
1638
+
1639
+ #if 0
1640
+ // sanity check
1641
+ std::unordered_map<llama_token, int> tmp;
1642
+ for (int i = 0; i < std::min<int>(ctx->penalty_last_n, ctx->prev.size()); ++i) {
1643
+ tmp[ctx->prev.rat(i)]++;
1644
+ }
1645
+
1646
+ assert(ctx->token_count == tmp);
1647
+ #endif
1648
+ }
1649
+
1650
+ static void llama_sampler_penalties_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
1651
+ auto * ctx = (llama_sampler_penalties *) smpl->ctx;
1652
+
1653
+ if ((ctx->penalty_last_n == 0) ||
1654
+ (ctx->penalty_repeat == 1.0f && ctx->penalty_freq == 0.0f && ctx->penalty_present == 0.0f)) {
1655
+ return;
1656
+ }
1657
+
1658
+ // Apply frequency and presence penalties to the cur_p
1659
+ for (size_t i = 0; i < cur_p->size; ++i) {
1660
+ const auto token_iter = ctx->token_count.find(cur_p->data[i].id);
1661
+ if (token_iter == ctx->token_count.end()) {
1662
+ continue;
1663
+ }
1664
+
1665
+ const int count = token_iter->second;
1666
+
1667
+ assert(count > 0 && count <= ctx->penalty_last_n);
1668
+
1669
+ // The academic publication that described this technique actually just only divided, but that would cause tokens with negative logits to become more likely, which is obviously wrong.
1670
+ // This is common fix for this problem, which is to multiply by the penalty instead of dividing.
1671
+ if (cur_p->data[i].logit <= 0) {
1672
+ cur_p->data[i].logit *= ctx->penalty_repeat;
1673
+ } else {
1674
+ cur_p->data[i].logit /= ctx->penalty_repeat;
1675
+ }
1676
+
1677
+ cur_p->data[i].logit -= float(count) * ctx->penalty_freq + float(count > 0) * ctx->penalty_present;
1678
+ }
1679
+
1680
+ cur_p->sorted = false;
1681
+ }
1682
+
1683
+ static void llama_sampler_penalties_reset(struct llama_sampler * smpl) {
1684
+ auto * ctx = (llama_sampler_penalties *) smpl->ctx;
1685
+ ctx->prev.clear();
1686
+ ctx->token_count.clear();
1687
+ }
1688
+
1689
+ static struct llama_sampler * llama_sampler_penalties_clone(const struct llama_sampler * smpl) {
1690
+ const auto * ctx = (const llama_sampler_penalties *) smpl->ctx;
1691
+ auto * result = llama_sampler_init_penalties(
1692
+ ctx->penalty_last_n,
1693
+ ctx->penalty_repeat,
1694
+ ctx->penalty_freq,
1695
+ ctx->penalty_present);
1696
+
1697
+ // copy the state
1698
+ {
1699
+ auto * result_ctx = (llama_sampler_penalties *) result->ctx;
1700
+
1701
+ result_ctx->prev = ctx->prev;
1702
+ }
1703
+
1704
+ return result;
1705
+ }
1706
+
1707
+ static void llama_sampler_penalties_free(struct llama_sampler * smpl) {
1708
+ delete (llama_sampler_penalties *) smpl->ctx;
1709
+ }
1710
+
1711
+ static struct llama_sampler_i llama_sampler_penalties_i = {
1712
+ /* .name = */ llama_sampler_penalties_name,
1713
+ /* .accept = */ llama_sampler_penalties_accept,
1714
+ /* .apply = */ llama_sampler_penalties_apply,
1715
+ /* .reset = */ llama_sampler_penalties_reset,
1716
+ /* .clone = */ llama_sampler_penalties_clone,
1717
+ /* .free = */ llama_sampler_penalties_free,
1718
+ };
1719
+
1720
+ struct llama_sampler * llama_sampler_init_penalties(
1721
+ int32_t penalty_last_n,
1722
+ float penalty_repeat,
1723
+ float penalty_freq,
1724
+ float penalty_present) {
1725
+ penalty_last_n = std::max(penalty_last_n, 0);
1726
+
1727
+ return llama_sampler_init(
1728
+ /* .iface = */ &llama_sampler_penalties_i,
1729
+ /* .ctx = */ new llama_sampler_penalties {
1730
+ /* .penalty_last_n = */ penalty_last_n,
1731
+ /* .penalty_repeat = */ penalty_repeat,
1732
+ /* .penalty_freq = */ penalty_freq,
1733
+ /* .penalty_present = */ penalty_present,
1734
+ /* .prev = */ ring_buffer<llama_token>(penalty_last_n),
1735
+ /* .token_count = */ {},
1736
+ }
1737
+ );
1738
+ }
1739
+
1740
+ // top-n-sigma
1741
+
1742
+ struct llama_sampler_top_n_sigma {
1743
+ const float n;
1744
+ };
1745
+
1746
+ static const char * llama_sampler_top_n_sigma_name(const struct llama_sampler * /*smpl*/) {
1747
+ return "top-n-sigma";
1748
+ }
1749
+
1750
+ static void llama_sampler_top_n_sigma_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
1751
+ const auto * ctx = (llama_sampler_top_n_sigma *) smpl->ctx;
1752
+
1753
+ if (ctx->n <= 0.0f || cur_p->size <= 1) {
1754
+ return;
1755
+ }
1756
+
1757
+ // find max logit and calculate mean
1758
+ float max = cur_p->data[0].logit;
1759
+ float logits_sum = 0;
1760
+ size_t valid_count = 0;
1761
+ for (size_t i = 0; i < cur_p->size; ++i) {
1762
+ // Only count non-negative infinity values
1763
+ if (cur_p->data[i].logit != -INFINITY) {
1764
+ if (cur_p->data[i].logit > max) {
1765
+ max = cur_p->data[i].logit;
1766
+ }
1767
+ logits_sum += cur_p->data[i].logit;
1768
+ valid_count++;
1769
+ }
1770
+ }
1771
+ float mean = valid_count > 0 ? logits_sum/valid_count : 0;
1772
+
1773
+ // calculate standard deviation
1774
+ float acc = 0;
1775
+ for (size_t i = 0; i < cur_p->size; ++i) {
1776
+ // Skip -infinity in std calculation
1777
+ if (cur_p->data[i].logit != -INFINITY) {
1778
+ acc += pow(cur_p->data[i].logit - mean, 2);
1779
+ }
1780
+ }
1781
+ float std = valid_count > 0 ? sqrt(acc/valid_count) : 0;
1782
+
1783
+ //apply mask
1784
+ for (size_t i = 0; i < cur_p->size; ++i) {
1785
+ if (cur_p->data[i].logit < max - (ctx->n * std)) {
1786
+ cur_p->data[i].logit = -INFINITY;
1787
+ }
1788
+ }
1789
+ llama_sampler_softmax_impl(cur_p);
1790
+ }
1791
+
1792
+ static struct llama_sampler * llama_sampler_top_n_sigma_clone(const struct llama_sampler * smpl) {
1793
+ const auto * ctx = (const llama_sampler_top_n_sigma *) smpl->ctx;
1794
+ return llama_sampler_init_top_n_sigma(ctx->n);
1795
+ }
1796
+
1797
+ static void llama_sampler_top_n_sigma_free(struct llama_sampler * smpl) {
1798
+ delete (llama_sampler_top_n_sigma *) smpl->ctx;
1799
+ }
1800
+
1801
+ static struct llama_sampler_i llama_sampler_top_n_sigma_i = {
1802
+ /* .name = */ llama_sampler_top_n_sigma_name,
1803
+ /* .accept = */ nullptr,
1804
+ /* .apply = */ llama_sampler_top_n_sigma_apply,
1805
+ /* .reset = */ nullptr,
1806
+ /* .clone = */ llama_sampler_top_n_sigma_clone,
1807
+ /* .free = */ llama_sampler_top_n_sigma_free,
1808
+ };
1809
+
1810
+ struct llama_sampler * llama_sampler_init_top_n_sigma(float n) {
1811
+ return llama_sampler_init(
1812
+ /* .iface = */ &llama_sampler_top_n_sigma_i,
1813
+ /* .ctx = */ new llama_sampler_top_n_sigma {
1814
+ /* .n = */ n,
1815
+ }
1816
+ );
1817
+ }
1818
+
1819
+ // DRY
1820
+
1821
+ struct llama_sampler_dry {
1822
+ int32_t total_context_size;
1823
+
1824
+ const float dry_multiplier;
1825
+ const float dry_base;
1826
+ const int32_t dry_allowed_length;
1827
+ const int32_t dry_penalty_last_n;
1828
+
1829
+ std::unordered_multimap<llama_token, std::vector<llama_token>> dry_processed_breakers;
1830
+ std::vector<int> dry_repeat_count;
1831
+ std::unordered_map<llama_token, int> dry_max_token_repeat;
1832
+ ring_buffer<llama_token> last_tokens;
1833
+ };
1834
+
1835
+ // Ported from Koboldcpp, original PR: https://github.com/LostRuins/koboldcpp/pull/982 (Original author: pi6am)
1836
+ static void get_overlapping_token_sequences(const llama_vocab & vocab, const std::string& str, std::unordered_multimap<llama_token, std::vector<llama_token>>& token_sequences, int max_tail_len = -1) {
1837
+ for (llama_token token_id = 0; token_id < (llama_token) vocab.n_tokens(); token_id++) {
1838
+ std::string word = vocab.detokenize({token_id}, true);
1839
+ if (word.find(str) != std::string::npos) {
1840
+ token_sequences.emplace(token_id, std::vector<llama_token>());
1841
+ } else {
1842
+ size_t word_len = word.size();
1843
+ size_t str_len = str.size();
1844
+ size_t pos = -1;
1845
+ while ((pos = word.find(str[0], pos + 1)) != std::string::npos) {
1846
+ bool match = true;
1847
+ size_t i;
1848
+ for (i = 1; i < str_len && i + pos < word_len; ++i) {
1849
+ if (word[pos + i] != str[i]) {
1850
+ match = false;
1851
+ break;
1852
+ }
1853
+ }
1854
+ if (match) {
1855
+ std::vector<llama_token> tokenization = vocab.tokenize(str.substr(i), false, false);
1856
+ if (max_tail_len >= 0 && tokenization.size() > (size_t)max_tail_len) {
1857
+ tokenization.resize(max_tail_len);
1858
+ }
1859
+
1860
+ // Ensure we don't already have a duplicate matching tokenization
1861
+ auto its = token_sequences.equal_range(token_id);
1862
+ bool found = false;
1863
+ for (auto it = its.first; it != its.second; ++it) {
1864
+ if (tokenization == it->second) {
1865
+ found = true;
1866
+ break;
1867
+ }
1868
+ }
1869
+ if (!found) {
1870
+ token_sequences.emplace(token_id, tokenization);
1871
+ }
1872
+ }
1873
+ }
1874
+ }
1875
+ }
1876
+ }
1877
+
1878
+ static const char * llama_sampler_dry_name(const struct llama_sampler * /*smpl*/) {
1879
+ return "dry";
1880
+ }
1881
+
1882
+ static void llama_sampler_dry_accept(struct llama_sampler * smpl, llama_token token) {
1883
+ auto * ctx = (llama_sampler_dry *) smpl->ctx;
1884
+ if (ctx->dry_multiplier == 0.0f || ctx->dry_base < 1.0f || ctx->dry_penalty_last_n == 0) {
1885
+ return;
1886
+ }
1887
+
1888
+ ctx->last_tokens.push_back(token);
1889
+ }
1890
+
1891
+ // Ported from Koboldcpp, original PR: https://github.com/LostRuins/koboldcpp/pull/982 (Original author: pi6am)
1892
+ static void llama_sampler_dry_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
1893
+ auto * ctx = (llama_sampler_dry *) smpl->ctx;
1894
+
1895
+ if (ctx->dry_multiplier == 0.0f || ctx->dry_base < 1.0f || ctx->dry_penalty_last_n == 0) {
1896
+ return;
1897
+ }
1898
+
1899
+ int32_t effective_dry_penalty_last_n = (ctx->dry_penalty_last_n == -1) ? ctx->total_context_size : std::max(ctx->dry_penalty_last_n, 0);
1900
+ int last_n_repeat = std::min(std::min((int)ctx->last_tokens.size(), effective_dry_penalty_last_n), ctx->total_context_size);
1901
+
1902
+ if (last_n_repeat <= ctx->dry_allowed_length) {
1903
+ return;
1904
+ }
1905
+
1906
+ ctx->dry_repeat_count.assign(last_n_repeat, 0);
1907
+ ctx->dry_max_token_repeat.clear();
1908
+
1909
+ // Step 1: Look for restart sequences to limit the maximum repetition length.
1910
+ // Work backwards through the context looking for any token that begins a restart sequence.
1911
+ //
1912
+ // The collection `restart_sequences` is a mapping from a "head" token to all "tail"
1913
+ // sequences that together comprise a restart sequence. This allows us to quickly check
1914
+ // whether each token is the head of a complete sequence. Most restart sequences are actually
1915
+ // a single token, and for these the "tail" is an empty vector.
1916
+ //
1917
+ // If the token is a "head", test all restart sequences that begin with this token
1918
+ // (there will often only be one sequence for each token, but if sequences like 'aaaq1' and
1919
+ // 'aaa1' are used as restart strings, both could start with 'aaa' when tokenized). The
1920
+ // longest matching sequence (if any) is used to limit the maximum repetition length.
1921
+ //
1922
+ // Note that in the case case of a short sequence contained in a longer one, this might fail to
1923
+ // find the smallest value for `rep_limit`. For example, if 'amniotic' and 'ni' are both used as
1924
+ // restart sequences, 'ni' will be found first, and since it's shorter it will fail to suppress
1925
+ // 'otic'. This is a minor issue since fully contained restart sequences are likely to be rare.
1926
+ //
1927
+ // This is theoretically worst-case O(N^2) for arbitrary restart sequences, which is why we
1928
+ // have already clamped the maximum tail sequence length when generating `restart_sequences`.
1929
+ // With clamping, this scan is O(N) in the context length.
1930
+
1931
+ int rep_limit = last_n_repeat;
1932
+ for (int i = 0; i < last_n_repeat; ++i) {
1933
+ llama_token token = ctx->last_tokens.rat(i);
1934
+ auto its = ctx->dry_processed_breakers.equal_range(token);
1935
+ if (its.first == ctx->dry_processed_breakers.end()) {
1936
+ continue;
1937
+ }
1938
+ int longest_match = -1;
1939
+ for (auto it = its.first; it != its.second; ++it) {
1940
+ // Note that (*it) does not contain the head character, so seq_len will be
1941
+ // the restart sequence length minus 1.
1942
+ // In the common case of a single-token restart sequence, (*it) will be empty
1943
+ // and we will trivially match.
1944
+ int seq_len = (int)it->second.size();
1945
+ if (seq_len > longest_match && seq_len <= (int)i) {
1946
+ bool match = true;
1947
+ for (int offset = 0; offset < seq_len; ++offset) {
1948
+ // The -1 when indexing `last_tokens` is because we already matched the head.
1949
+ if (it->second[offset] != ctx->last_tokens.rat(i - offset - 1)) {
1950
+ match = false;
1951
+ break;
1952
+ }
1953
+ }
1954
+ if (match) {
1955
+ longest_match = seq_len;
1956
+ }
1957
+ }
1958
+ }
1959
+ if (longest_match >= 0) {
1960
+ // We found a restart sequence starting `i` tokens from the end and continuing for
1961
+ // `longest_match` tokens.
1962
+ rep_limit = i - longest_match;
1963
+ break;
1964
+ }
1965
+ }
1966
+ if (rep_limit < ctx->dry_allowed_length) {
1967
+ return;
1968
+ }
1969
+
1970
+ // Step 2: Iterate in reverse over the last N tokens of the context, using the "Z-algorithm" (in
1971
+ // the reverse direction) to efficiently compute the positions and lengths of suffixes appearing
1972
+ // elsewhere in the context. We limit the suffix length to `rep_limit` to respect restart sequences.
1973
+ //
1974
+ // This algorithm is not currently documented on Wikipedia, but there is a clear description here:
1975
+ // https://ivanyu.me/blog/2014/10/15/z-algorithm/
1976
+ //
1977
+ // The code below is adapted from the public domain implementation by the same author here:
1978
+ // https://github.com/ivanyu/string-algorithms/blob/master/z_algorithm.py
1979
+ //
1980
+ // Example:
1981
+ // Last N tokens: a b c c b c y a b c
1982
+ // Repeat counts: 0 0 3 1 0 2 0 0 0 0
1983
+ // ^
1984
+ // This `3` means that the last three tokens of the context (a b c) also appear here.
1985
+ //
1986
+ // This step is worst case O(N) since the Z-algorithm is linear, despite the appearance of nested
1987
+ // for/while loops. This can be seen by observing that the `lt` and `rt` bounds are set after each
1988
+ // repeated suffix is detected (i.e. after each while loop when n > 0). These bound variables
1989
+ // ensure that the inner while loops only examine each token in the context once as the outer
1990
+ // for loop iterates over the context.
1991
+
1992
+ {
1993
+ const int last = last_n_repeat - 1;
1994
+ int rt = 0, lt = 0;
1995
+
1996
+ for (int k = 1; k < last_n_repeat; ++k) {
1997
+ if (k > rt) {
1998
+ // If k is outside the current Z-box, do naive computation.
1999
+ int n = 0;
2000
+ while (n + k < last_n_repeat && ctx->last_tokens.rat(n) == ctx->last_tokens.rat(n+k)) {
2001
+ ++n;
2002
+ }
2003
+ ctx->dry_repeat_count[last - k] = std::min(n, rep_limit);
2004
+ if (n > 0) {
2005
+ lt = k;
2006
+ rt = k + n - 1;
2007
+ }
2008
+ } else {
2009
+ // If k is inside the current Z-box, consider two cases.
2010
+
2011
+ int p = k - lt; // Pair index.
2012
+ int right_part_len = rt - k + 1;
2013
+
2014
+ if (ctx->dry_repeat_count[last - p] < right_part_len) {
2015
+ int n = std::min(ctx->dry_repeat_count[last - p], rep_limit);
2016
+ ctx->dry_repeat_count[last - k] = n;
2017
+ } else {
2018
+ int i = rt + 1;
2019
+ while (i < last_n_repeat && ctx->last_tokens.rat(i) == ctx->last_tokens.rat(i - k)) {
2020
+ i += 1;
2021
+ }
2022
+
2023
+ int n = std::min(i - k, rep_limit);
2024
+ ctx->dry_repeat_count[last - k] = n;
2025
+ lt = k;
2026
+ rt = i - 1;
2027
+ }
2028
+ }
2029
+ }
2030
+ }
2031
+
2032
+ // Step 3: Iterate over dry_repeat_count and last_tokens, examining the maximum repeat length
2033
+ // that would be generated by emitting each new token that would extend a sequence.
2034
+ //
2035
+ // Following the same example as above:
2036
+ // Last N tokens: a b c c b c y a b c
2037
+ // Repeat counts: 0 0 3 1 0 2 0 0 0 0
2038
+ //
2039
+ // For each non-zero, look ahead one token. This token, if emitted, would extend the repetition.
2040
+ // c: 3 -> 4 (from `a b c` to `a b c c`)
2041
+ // b: 1 -> 2 (from `c` to `c b`)
2042
+ // y: 2 -> 3 (from `b c` to `b c y`)
2043
+
2044
+ for (int i = 0; i < last_n_repeat - 1; ++i) {
2045
+ int repeat_len = ctx->dry_repeat_count[i];
2046
+ if (repeat_len >= ctx->dry_allowed_length) {
2047
+ // This token ends a repeat, so the next token would continue one.
2048
+ // By convention, the value of `repeat_len` only includes the tokens currently
2049
+ // in the context, not the new token that would be added.
2050
+ llama_token token = ctx->last_tokens.rat(last_n_repeat - 2 - i);
2051
+ // Track the maximum sequence ending in this token.
2052
+ const auto& it = ctx->dry_max_token_repeat.find(token);
2053
+ if (it == ctx->dry_max_token_repeat.end() || it->second < repeat_len) {
2054
+ ctx->dry_max_token_repeat[token] = repeat_len;
2055
+ }
2056
+ }
2057
+ }
2058
+
2059
+ // Step 4: Apply logit penalties based on the maximum repeat length for relevant tokens.
2060
+
2061
+ // Prevent floating point overflow in `pow(penalty_base, exponent)` by clamping to `max_exponent`.
2062
+ // Compute it from `penalty_base` and the approximate log of `std::numeric_limits<float>::max()`
2063
+ const float FLOAT_MAX_LOG = 88.7228391f;
2064
+ int max_exponent = 0;
2065
+ if (ctx->dry_base > 1.000001f) {
2066
+ max_exponent = FLOAT_MAX_LOG / std::log(ctx->dry_base);
2067
+ }
2068
+
2069
+ for (size_t i = 0; i < cur_p->size; ++i) {
2070
+ const auto& af_kvp = ctx->dry_max_token_repeat.find(cur_p->data[i].id);
2071
+ if (af_kvp != ctx->dry_max_token_repeat.end()) {
2072
+ // Check all sequence breakers starting with this token
2073
+ auto range = ctx->dry_processed_breakers.equal_range(cur_p->data[i].id);
2074
+ bool is_single_token_breaker = false;
2075
+
2076
+ for (auto it = range.first; it != range.second; ++it) {
2077
+ if (it->second.empty()) {
2078
+ is_single_token_breaker = true;
2079
+ break;
2080
+ }
2081
+ }
2082
+
2083
+ // Apply penalty only if it's not a single-token sequence breaker
2084
+ if (!is_single_token_breaker) {
2085
+ int repeat_exp = af_kvp->second - ctx->dry_allowed_length;
2086
+ if (max_exponent > 0 && repeat_exp > max_exponent) {
2087
+ repeat_exp = max_exponent;
2088
+ }
2089
+ float penalty = ctx->dry_multiplier * std::pow(ctx->dry_base, repeat_exp);
2090
+ cur_p->data[i].logit -= penalty;
2091
+ }
2092
+ }
2093
+ }
2094
+
2095
+ cur_p->sorted = false;
2096
+ }
2097
+
2098
+ static void llama_sampler_dry_reset(struct llama_sampler * smpl) {
2099
+ auto * ctx = (llama_sampler_dry *) smpl->ctx;
2100
+ ctx->last_tokens.clear();
2101
+ ctx->dry_repeat_count.clear();
2102
+ ctx->dry_max_token_repeat.clear();
2103
+ }
2104
+
2105
+ static struct llama_sampler * llama_sampler_dry_clone(const struct llama_sampler * smpl) {
2106
+ const auto * ctx = (llama_sampler_dry *) smpl->ctx;
2107
+
2108
+ llama_vocab dummy_vocab;
2109
+
2110
+ // dummy vocab is passed because it is only needed for raw sequence breaker processing, which we have already done and will simply be copying
2111
+ auto * result = llama_sampler_init_dry(&dummy_vocab, ctx->total_context_size, ctx->dry_multiplier, ctx->dry_base, ctx->dry_allowed_length, ctx->dry_penalty_last_n, NULL, 0);
2112
+
2113
+ // Copy the state, including the processed breakers
2114
+ {
2115
+ auto * result_ctx = (llama_sampler_dry *) result->ctx;
2116
+ result_ctx->dry_processed_breakers = ctx->dry_processed_breakers;
2117
+ result_ctx->dry_repeat_count = ctx->dry_repeat_count;
2118
+ result_ctx->dry_max_token_repeat = ctx->dry_max_token_repeat;
2119
+ result_ctx->last_tokens = ctx->last_tokens;
2120
+ }
2121
+
2122
+ return result;
2123
+ }
2124
+
2125
+ static void llama_sampler_dry_free(struct llama_sampler * smpl) {
2126
+ delete (llama_sampler_dry *) smpl->ctx;
2127
+ }
2128
+
2129
+ static struct llama_sampler_i llama_sampler_dry_i = {
2130
+ /* .name = */ llama_sampler_dry_name,
2131
+ /* .accept = */ llama_sampler_dry_accept,
2132
+ /* .apply = */ llama_sampler_dry_apply,
2133
+ /* .reset = */ llama_sampler_dry_reset,
2134
+ /* .clone = */ llama_sampler_dry_clone,
2135
+ /* .free = */ llama_sampler_dry_free,
2136
+ };
2137
+
2138
+ struct llama_sampler * llama_sampler_init_dry(const struct llama_vocab * vocab, int32_t context_size, float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const char** seq_breakers, size_t num_breakers) {
2139
+ int32_t effective_dry_penalty_last_n = (dry_penalty_last_n == -1) ? context_size : std::max(dry_penalty_last_n, 0);
2140
+ std::unordered_multimap<llama_token, std::vector<llama_token>> processed_breakers;
2141
+ const int MAX_CHAR_LEN = 40;
2142
+ const int MAX_SEQ_LEN = 20;
2143
+
2144
+ const bool dry_enabled = (dry_multiplier != 0.0f && dry_base >= 1.0f && dry_penalty_last_n != 0);
2145
+
2146
+ if (dry_enabled && seq_breakers != nullptr && num_breakers > 0) {
2147
+ // Process sequence breakers
2148
+ for (size_t i = 0; i < num_breakers; ++i) {
2149
+ if (seq_breakers[i] == nullptr || std::strlen(seq_breakers[i]) == 0) {
2150
+ LLAMA_LOG_WARN("skipping null or empty DRY sequence breaker at index %zu\n", i);
2151
+ continue;
2152
+ }
2153
+
2154
+ std::string sequence_break(seq_breakers[i]);
2155
+ if (sequence_break.empty()) {
2156
+ LLAMA_LOG_WARN("skipping empty DRY sequence breaker\n");
2157
+ continue;
2158
+ }
2159
+
2160
+ if (sequence_break.size() > MAX_CHAR_LEN) {
2161
+ LLAMA_LOG_WARN("truncating DRY sequence breaker to %d characters\n", MAX_CHAR_LEN);
2162
+ sequence_break.resize(MAX_CHAR_LEN);
2163
+ }
2164
+
2165
+ get_overlapping_token_sequences(*vocab, sequence_break, processed_breakers, MAX_SEQ_LEN);
2166
+ }
2167
+ }
2168
+
2169
+ return llama_sampler_init(
2170
+ /* .iface = */ &llama_sampler_dry_i,
2171
+ /* .ctx = */ new llama_sampler_dry {
2172
+ /* .total_context_size = */ context_size,
2173
+ /* .dry_multiplier = */ dry_multiplier,
2174
+ /* .dry_base = */ dry_base,
2175
+ /* .dry_allowed_length = */ dry_allowed_length,
2176
+ /* .dry_penalty_last_n = */ dry_penalty_last_n,
2177
+ /* .dry_processed_breakers = */ std::move(processed_breakers),
2178
+ /* .dry_repeat_count = */ dry_enabled ? std::vector<int>(effective_dry_penalty_last_n, 0) : std::vector<int>{},
2179
+ /* .dry_max_token_repeat = */ {},
2180
+ /* .last_tokens = */ dry_enabled ? ring_buffer<llama_token>(effective_dry_penalty_last_n) : ring_buffer<llama_token>(0),
2181
+ }
2182
+ );
2183
+ }
2184
+
2185
+ // wrapper for test-sampling.cpp
2186
+ struct llama_sampler * llama_sampler_init_dry_testing(int32_t context_size, float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const std::vector<std::vector<llama_token>>& seq_breakers) {
2187
+ llama_vocab dummy_vocab;
2188
+ auto * result = llama_sampler_init_dry(&dummy_vocab, context_size, dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n, NULL, 0);
2189
+ auto * ctx = (llama_sampler_dry *) result->ctx;
2190
+
2191
+ // Process the token-based sequence breakers
2192
+ ctx->dry_processed_breakers.clear();
2193
+ if (seq_breakers.empty()) {
2194
+ LLAMA_LOG_WARN("empty DRY sequence breakers list in llama_sampler_init_dry_testing\n");
2195
+ } else {
2196
+ for (const auto& breaker : seq_breakers) {
2197
+ if (breaker.empty()) {
2198
+ LLAMA_LOG_WARN("skipping DRY empty sequence breaker\n");
2199
+ continue;
2200
+ }
2201
+ llama_token head_token = breaker[0];
2202
+ std::vector<llama_token> tail_tokens(breaker.begin() + 1, breaker.end());
2203
+ ctx->dry_processed_breakers.emplace(head_token, std::move(tail_tokens));
2204
+ }
2205
+
2206
+ if (ctx->dry_processed_breakers.empty()) {
2207
+ LLAMA_LOG_WARN("no valid DRY sequence breakers processed in llama_sampler_init_dry_testing\n");
2208
+ }
2209
+ }
2210
+
2211
+ return result;
2212
+ }
2213
+
2214
+ // logit-bias
2215
+
2216
+ struct llama_sampler_logit_bias {
2217
+ const int32_t n_vocab;
2218
+
2219
+ const std::vector<llama_logit_bias> logit_bias;
2220
+
2221
+ std::vector<llama_logit_bias> to_search;
2222
+ };
2223
+
2224
+ static const char * llama_sampler_logit_bias_name(const struct llama_sampler * /*smpl*/) {
2225
+ return "logit-bias";
2226
+ }
2227
+
2228
+ static void llama_sampler_logit_bias_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
2229
+ auto * ctx = (llama_sampler_logit_bias *) smpl->ctx;
2230
+
2231
+ if (ctx->logit_bias.empty()) {
2232
+ return;
2233
+ }
2234
+
2235
+ ctx->to_search.clear();
2236
+
2237
+ // update the candidates that have not been shuffled in the vocabulary (i.e. idx == id)
2238
+ for (const auto & lb : ctx->logit_bias) {
2239
+ if (lb.token >= 0 && cur_p->size > (size_t) lb.token && cur_p->data[lb.token].id == lb.token) {
2240
+ cur_p->data[lb.token].logit += lb.bias;
2241
+ } else {
2242
+ ctx->to_search.push_back(lb);
2243
+ }
2244
+ }
2245
+
2246
+ if (ctx->to_search.empty()) {
2247
+ return;
2248
+ }
2249
+
2250
+ // search for the remaining candidates that were not found in the previous step
2251
+ for (size_t i = 0; i < cur_p->size; ++i) {
2252
+ for (const auto & lb : ctx->to_search) {
2253
+ if (cur_p->data[i].id == lb.token) {
2254
+ cur_p->data[i].logit += lb.bias;
2255
+ break;
2256
+ }
2257
+ }
2258
+ }
2259
+ }
2260
+
2261
+ static struct llama_sampler * llama_sampler_logit_bias_clone(const struct llama_sampler * smpl) {
2262
+ const auto * ctx = (const llama_sampler_logit_bias *) smpl->ctx;
2263
+ return llama_sampler_init_logit_bias(ctx->n_vocab, ctx->logit_bias.size(), ctx->logit_bias.data());
2264
+ }
2265
+
2266
+ static void llama_sampler_logit_bias_free(struct llama_sampler * smpl) {
2267
+ delete (llama_sampler_logit_bias *) smpl->ctx;
2268
+ }
2269
+
2270
+ static struct llama_sampler_i llama_sampler_logit_bias_i = {
2271
+ /* .name = */ llama_sampler_logit_bias_name,
2272
+ /* .accept = */ nullptr,
2273
+ /* .apply = */ llama_sampler_logit_bias_apply,
2274
+ /* .reset = */ nullptr,
2275
+ /* .clone = */ llama_sampler_logit_bias_clone,
2276
+ /* .free = */ llama_sampler_logit_bias_free,
2277
+ };
2278
+
2279
+ struct llama_sampler * llama_sampler_init_logit_bias(
2280
+ int32_t n_vocab,
2281
+ int32_t n_logit_bias,
2282
+ const llama_logit_bias * logit_bias) {
2283
+ return llama_sampler_init(
2284
+ /* .iface = */ &llama_sampler_logit_bias_i,
2285
+ /* .ctx = */ new llama_sampler_logit_bias {
2286
+ /* .n_vocab = */ n_vocab,
2287
+ /* .logit_bias = */ std::vector<llama_logit_bias>(logit_bias, logit_bias + n_logit_bias),
2288
+ /* .to_search = */ {},
2289
+ }
2290
+ );
2291
+ }
2292
+
2293
+ // infill
2294
+
2295
+ //#define GGML_DEBUG_SAMPLER_INFILL
2296
+
2297
+ struct llama_sampler_infill {
2298
+ const struct llama_vocab * vocab;
2299
+
2300
+ std::vector<char> buf0;
2301
+ std::vector<char> buf1;
2302
+ };
2303
+
2304
+ static const char * llama_sampler_infill_name(const struct llama_sampler * /*smpl*/) {
2305
+ return "infill";
2306
+ }
2307
+
2308
+ static void llama_sampler_infill_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {
2309
+ auto * ctx = (llama_sampler_infill *) smpl->ctx;
2310
+
2311
+ llama_sampler_softmax_impl(cur_p);
2312
+
2313
+ #if defined(GGML_DEBUG_SAMPLER_INFILL)
2314
+ #define LOG_DBG_CUR LLAMA_LOG_DEBUG
2315
+ #else
2316
+ #define LOG_DBG_CUR(...)
2317
+ #endif
2318
+
2319
+ for (size_t i = 0; i < cur_p->size; ++i) {
2320
+ LOG_DBG_CUR("%s: cur_p[%3zu] = { id: %6d, p: %.6f, logit: %6.3f }\n", __func__, i, cur_p->data[i].id, cur_p->data[i].p, cur_p->data[i].logit);
2321
+ }
2322
+
2323
+ float p_txt_sum = 0.0f;
2324
+ float p_eog_sum = 0.0f;
2325
+
2326
+ for (size_t i = 0; i < cur_p->size; ++i) {
2327
+ if (ctx->vocab->is_eog(cur_p->data[i].id)) {
2328
+ p_eog_sum += cur_p->data[i].p;
2329
+ } else {
2330
+ p_txt_sum += cur_p->data[i].p;
2331
+ }
2332
+ }
2333
+
2334
+ const float rat = p_eog_sum == 0.0 ? INFINITY : p_txt_sum / p_eog_sum; GGML_UNUSED(rat);
2335
+
2336
+ LOG_DBG_CUR("%s: p_txt_sum = %.2f, p_eog_sum = %.2f, rat = %.2f, n = %zu\n", __func__, p_txt_sum, p_eog_sum, rat, cur_p->size);
2337
+
2338
+ if (3*p_eog_sum*cur_p->size > p_txt_sum) {
2339
+ LOG_DBG_CUR("%s: the ratio p_txt/p_eog = %.2f is too low -> sampling EOG\n", __func__, p_txt_sum/p_eog_sum);
2340
+
2341
+ // keep just the EOG tokens
2342
+ const auto size_org = cur_p->size;
2343
+
2344
+ cur_p->size = 0;
2345
+
2346
+ float p_sum = 0.0f;
2347
+
2348
+ for (size_t i = 0; i < size_org; ++i) {
2349
+ if (ctx->vocab->is_eog(cur_p->data[i].id)) {
2350
+ p_sum += cur_p->data[i].p;
2351
+
2352
+ cur_p->data[cur_p->size++] = cur_p->data[i];
2353
+ }
2354
+ }
2355
+
2356
+ // normalize probs
2357
+ for (size_t i = 0; i < cur_p->size; ++i) {
2358
+ cur_p->data[i].p /= p_sum;
2359
+ }
2360
+
2361
+ return;
2362
+ }
2363
+
2364
+ size_t n_combined = 0; GGML_UNUSED(n_combined);
2365
+
2366
+ // combine tokens with common prefix
2367
+ for (size_t i0 = 0; i0 < cur_p->size; ++i0) {
2368
+ for (size_t i1 = 0; i1 < cur_p->size; ++i1) {
2369
+ if (cur_p->data[i0].logit == -INFINITY) {
2370
+ break;
2371
+ }
2372
+
2373
+ if (i0 == i1 || cur_p->data[i1].logit == -INFINITY) {
2374
+ continue;
2375
+ }
2376
+
2377
+ int len0 = ctx->vocab->token_to_piece(cur_p->data[i0].id, ctx->buf0.data(), ctx->buf0.size(), 0, false);
2378
+ if (len0 < 0) {
2379
+ ctx->buf0.resize(len0);
2380
+ len0 = ctx->vocab->token_to_piece(cur_p->data[i0].id, ctx->buf0.data(), ctx->buf0.size(), 0, false);
2381
+ assert(len0 > 0);
2382
+ }
2383
+
2384
+ int len1 = ctx->vocab->token_to_piece(cur_p->data[i1].id, ctx->buf1.data(), ctx->buf1.size(), 0, false);
2385
+ if (len1 < 0) {
2386
+ ctx->buf1.resize(len1);
2387
+ len1 = ctx->vocab->token_to_piece(cur_p->data[i1].id, ctx->buf1.data(), ctx->buf1.size(), 0, false);
2388
+ assert(len1 > 0);
2389
+ }
2390
+
2391
+ // token i0 is a prefix of token i1
2392
+ if (len0 > 0 && len0 <= len1 && memcmp(ctx->buf0.data(), ctx->buf1.data(), len0) == 0) {
2393
+ int dst = i0;
2394
+ int src = i1;
2395
+
2396
+ // merge into the token with higher probability
2397
+ if (cur_p->data[i1].p > cur_p->data[i0].p) {
2398
+ std::swap(dst, src);
2399
+ }
2400
+
2401
+ cur_p->data[dst].p += cur_p->data[src].p;
2402
+ cur_p->data[src].logit = -INFINITY;
2403
+ cur_p->data[src].p = 0.0f;
2404
+
2405
+ n_combined++;
2406
+ }
2407
+ }
2408
+ }
2409
+
2410
+ size_t n_non_eog = 0;
2411
+
2412
+ size_t size_org = cur_p->size;
2413
+
2414
+ float p_sum = 0.0f;
2415
+ float thold = 0.2f;
2416
+
2417
+ cur_p->size = 0;
2418
+
2419
+ LOG_DBG_CUR("%s: n_combined = %zu, applying thold = %.3f\n", __func__, n_combined, thold);
2420
+
2421
+ for (size_t i = 0; i < size_org; ++i) {
2422
+ const bool is_eog = ctx->vocab->is_eog(cur_p->data[i].id);
2423
+
2424
+ if (cur_p->data[i].p < thold && !is_eog) {
2425
+ continue;
2426
+ }
2427
+
2428
+ if (!is_eog) {
2429
+ ++n_non_eog;
2430
+ }
2431
+
2432
+ p_sum += cur_p->data[i].p;
2433
+
2434
+ // keep this token
2435
+ cur_p->data[cur_p->size++] = cur_p->data[i];
2436
+ }
2437
+
2438
+ LOG_DBG_CUR("%s: n_non_eog = %zu\n", __func__, n_non_eog);
2439
+
2440
+ // if no non-EOG tokens are left -> reduce cur_p to single EOT token
2441
+ if (n_non_eog == 0) {
2442
+ cur_p->size = 1;
2443
+ cur_p->data[0].id = ctx->vocab->token_eot();
2444
+ cur_p->data[0].logit = 1.0f;
2445
+
2446
+ return;
2447
+ }
2448
+
2449
+ // normalize probs
2450
+ for (size_t i = 0; i < cur_p->size; ++i) {
2451
+ cur_p->data[i].p /= p_sum;
2452
+
2453
+ LOG_DBG_CUR("%s: cur_p[%3zu] = { id: %6d, p: %.6f, logit: %6.3f }\n", __func__, i, cur_p->data[i].id, cur_p->data[i].p, cur_p->data[i].logit);
2454
+ }
2455
+
2456
+ size_org = cur_p->size;
2457
+ p_sum = 0.0f;
2458
+ thold = 1.0/(n_non_eog + 1);
2459
+
2460
+ cur_p->size = 0;
2461
+
2462
+ LOG_DBG_CUR("%s: applying thold = %.3f\n", __func__, thold);
2463
+
2464
+ for (size_t i = 0; i < size_org; ++i) {
2465
+ const bool is_eog = ctx->vocab->is_eog(cur_p->data[i].id);
2466
+
2467
+ if (cur_p->data[i].p < thold && !is_eog) {
2468
+ continue;
2469
+ }
2470
+
2471
+ p_sum += cur_p->data[i].p;
2472
+
2473
+ cur_p->data[cur_p->size++] = cur_p->data[i];
2474
+ }
2475
+
2476
+ // normalize probs
2477
+ for (size_t i = 0; i < cur_p->size; ++i) {
2478
+ cur_p->data[i].p /= p_sum;
2479
+
2480
+ LOG_DBG_CUR("%s: cur_p[%3zu] = { id: %6d, p: %.6f, logit: %6.3f }\n", __func__, i, cur_p->data[i].id, cur_p->data[i].p, cur_p->data[i].logit);
2481
+ }
2482
+
2483
+ #undef LOG_DBG_CUR
2484
+ }
2485
+
2486
+ static struct llama_sampler * llama_sampler_infill_clone(const struct llama_sampler * smpl) {
2487
+ const auto * ctx = (const llama_sampler_infill *) smpl->ctx;
2488
+ return llama_sampler_init_infill(ctx->vocab);
2489
+ }
2490
+
2491
+ static void llama_sampler_infill_free(struct llama_sampler * smpl) {
2492
+ delete (llama_sampler_infill *) smpl->ctx;
2493
+ }
2494
+
2495
+ static struct llama_sampler_i llama_sampler_infill_i = {
2496
+ /* .name = */ llama_sampler_infill_name,
2497
+ /* .accept = */ nullptr,
2498
+ /* .apply = */ llama_sampler_infill_apply,
2499
+ /* .reset = */ nullptr,
2500
+ /* .clone = */ llama_sampler_infill_clone,
2501
+ /* .free = */ llama_sampler_infill_free,
2502
+ };
2503
+
2504
+ struct llama_sampler * llama_sampler_init_infill(const struct llama_vocab * vocab) {
2505
+ return llama_sampler_init(
2506
+ /* .iface = */ &llama_sampler_infill_i,
2507
+ /* .ctx = */ new llama_sampler_infill {
2508
+ /* .vocab = */ vocab,
2509
+ /* .buf0 = */ std::vector<char>(512),
2510
+ /* .buf1 = */ std::vector<char>(512),
2511
+ }
2512
+ );
2513
+ }
2514
+
2515
+ // utils
2516
+
2517
+ uint32_t llama_sampler_get_seed(const struct llama_sampler * smpl) {
2518
+ if (smpl->iface == &llama_sampler_dist_i) {
2519
+ return ((const llama_sampler_dist *) smpl->ctx)->seed_cur;
2520
+ }
2521
+
2522
+ if (smpl->iface == &llama_sampler_mirostat_i) {
2523
+ return ((const llama_sampler_mirostat *) smpl->ctx)->seed_cur;
2524
+ }
2525
+
2526
+ if (smpl->iface == &llama_sampler_mirostat_v2_i) {
2527
+ return ((const llama_sampler_mirostat_v2 *) smpl->ctx)->seed_cur;
2528
+ }
2529
+
2530
+ if (smpl->iface == &llama_sampler_chain_i) {
2531
+ const auto * ctx = (const llama_sampler_chain *) smpl->ctx;
2532
+ for (auto it = ctx->samplers.rbegin(); it != ctx->samplers.rend(); ++it) {
2533
+ const uint32_t seed = llama_sampler_get_seed(*it);
2534
+ if (seed != LLAMA_DEFAULT_SEED) {
2535
+ return seed;
2536
+ }
2537
+ }
2538
+ }
2539
+
2540
+ return LLAMA_DEFAULT_SEED;
2541
+ }
2542
+
2543
+ // perf
2544
+
2545
+ struct llama_perf_sampler_data llama_perf_sampler(const struct llama_sampler * chain) {
2546
+ struct llama_perf_sampler_data data = {};
2547
+
2548
+ if (chain == nullptr || chain->iface != &llama_sampler_chain_i) {
2549
+ GGML_ABORT("%s: invalid sampler passed - requires a sampler created with llama_sampler_chain_init()\n", __func__);
2550
+ }
2551
+
2552
+ const auto * ctx = (const struct llama_sampler_chain *) chain->ctx;
2553
+
2554
+ data.t_sample_ms = 1e-3 * ctx->t_sample_us;
2555
+ data.n_sample = std::max(0, ctx->n_sample);
2556
+
2557
+ return data;
2558
+ }
2559
+
2560
+ void llama_perf_sampler_print(const struct llama_sampler * chain) {
2561
+ const auto data = llama_perf_sampler(chain);
2562
+
2563
+ LLAMA_LOG_INFO("%s: sampling time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n",
2564
+ __func__, data.t_sample_ms, data.n_sample, data.t_sample_ms / data.n_sample, 1e3 / data.t_sample_ms * data.n_sample);
2565
+ }
2566
+
2567
+ void llama_perf_sampler_reset(struct llama_sampler * chain) {
2568
+ if (chain == nullptr || chain->iface != &llama_sampler_chain_i) {
2569
+ GGML_ABORT("%s: invalid sampler passed - requires a sampler created with llama_sampler_chain_init()\n", __func__);
2570
+ }
2571
+
2572
+ auto * ctx = (struct llama_sampler_chain *) chain->ctx;
2573
+
2574
+ ctx->t_sample_us = ctx->n_sample = 0;
2575
+ }