cohere-transcribe 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (480) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +21 -0
  3. data/LICENSE.txt +201 -0
  4. data/NOTICE +5 -0
  5. data/README.md +265 -0
  6. data/THIRD_PARTY_NOTICES.md +67 -0
  7. data/exe/cohere-transcribe +8 -0
  8. data/exe/cohere-transcribe-doctor +8 -0
  9. data/ext/cohere_transcribe_native/CMakeLists.txt +262 -0
  10. data/ext/cohere_transcribe_native/README.md +26 -0
  11. data/ext/cohere_transcribe_native/audio_abi.cpp +1416 -0
  12. data/ext/cohere_transcribe_native/audio_exports.macos +5 -0
  13. data/ext/cohere_transcribe_native/audio_exports.map +10 -0
  14. data/ext/cohere_transcribe_native/cohere_abi.cpp +1209 -0
  15. data/ext/cohere_transcribe_native/exports.macos +40 -0
  16. data/ext/cohere_transcribe_native/exports.map +45 -0
  17. data/ext/cohere_transcribe_native/extconf.rb +89 -0
  18. data/ext/cohere_transcribe_native/test/abi_smoke.rb +232 -0
  19. data/ext/cohere_transcribe_native/test/audio_matrix_smoke.cpp +121 -0
  20. data/ext/cohere_transcribe_native/test/audio_reliability_smoke.cpp +198 -0
  21. data/ext/cohere_transcribe_native/test/native_abi_reliability_smoke.cpp +186 -0
  22. data/ext/cohere_transcribe_native/test/native_batch_projection_probe.rb +81 -0
  23. data/ext/cohere_transcribe_native/test/native_cancellation_smoke.cpp +194 -0
  24. data/lib/cohere/transcribe/alignment/ATTRIBUTION.md +23 -0
  25. data/lib/cohere/transcribe/alignment/LICENSE.ctc-forced-aligner +407 -0
  26. data/lib/cohere/transcribe/alignment/LICENSE.torchaudio +25 -0
  27. data/lib/cohere/transcribe/alignment/LICENSE.uroman +26 -0
  28. data/lib/cohere/transcribe/alignment/aligner.rb +476 -0
  29. data/lib/cohere/transcribe/alignment/ctc.rb +224 -0
  30. data/lib/cohere/transcribe/alignment/text.rb +237 -0
  31. data/lib/cohere/transcribe/alignment/uroman_data.rb +4950 -0
  32. data/lib/cohere/transcribe/api.rb +173 -0
  33. data/lib/cohere/transcribe/asr/batching.rb +472 -0
  34. data/lib/cohere/transcribe/asr/failure_policy.rb +112 -0
  35. data/lib/cohere/transcribe/asr/native.rb +676 -0
  36. data/lib/cohere/transcribe/audio/ATTRIBUTION.md +8 -0
  37. data/lib/cohere/transcribe/audio/LICENSE.auditok +21 -0
  38. data/lib/cohere/transcribe/audio/decoder.rb +315 -0
  39. data/lib/cohere/transcribe/audio/ffmpeg_native.rb +248 -0
  40. data/lib/cohere/transcribe/audio/segmentation.rb +260 -0
  41. data/lib/cohere/transcribe/cli.rb +727 -0
  42. data/lib/cohere/transcribe/configuration.rb +282 -0
  43. data/lib/cohere/transcribe/constants.rb +14 -0
  44. data/lib/cohere/transcribe/dense_converter.rb +548 -0
  45. data/lib/cohere/transcribe/doctor.rb +576 -0
  46. data/lib/cohere/transcribe/errors.rb +57 -0
  47. data/lib/cohere/transcribe/gguf_writer.rb +268 -0
  48. data/lib/cohere/transcribe/hub.rb +436 -0
  49. data/lib/cohere/transcribe/input.rb +110 -0
  50. data/lib/cohere/transcribe/licenses/crispasr.txt +21 -0
  51. data/lib/cohere/transcribe/loader.rb +128 -0
  52. data/lib/cohere/transcribe/model_identity.rb +440 -0
  53. data/lib/cohere/transcribe/output/publication.rb +1118 -0
  54. data/lib/cohere/transcribe/output/rendering.rb +105 -0
  55. data/lib/cohere/transcribe/output/timing.rb +86 -0
  56. data/lib/cohere/transcribe/python_text.rb +70 -0
  57. data/lib/cohere/transcribe/pytorch_checkpoint.rb +1180 -0
  58. data/lib/cohere/transcribe/runtime/engine.rb +1676 -0
  59. data/lib/cohere/transcribe/runtime/model_provider.rb +390 -0
  60. data/lib/cohere/transcribe/runtime/precision.rb +57 -0
  61. data/lib/cohere/transcribe/runtime/preparation.rb +215 -0
  62. data/lib/cohere/transcribe/runtime/resources.rb +165 -0
  63. data/lib/cohere/transcribe/runtime/word_pipeline.rb +364 -0
  64. data/lib/cohere/transcribe/safetensors.rb +579 -0
  65. data/lib/cohere/transcribe/state/checkpoint.rb +224 -0
  66. data/lib/cohere/transcribe/state/contracts.rb +141 -0
  67. data/lib/cohere/transcribe/state/io.rb +727 -0
  68. data/lib/cohere/transcribe/state/locking.rb +211 -0
  69. data/lib/cohere/transcribe/state/manifest.rb +155 -0
  70. data/lib/cohere/transcribe/state.rb +7 -0
  71. data/lib/cohere/transcribe/types.rb +535 -0
  72. data/lib/cohere/transcribe/vad/ATTRIBUTION.md +14 -0
  73. data/lib/cohere/transcribe/vad/LICENSE.faster-whisper +21 -0
  74. data/lib/cohere/transcribe/vad/LICENSE.silero-vad +21 -0
  75. data/lib/cohere/transcribe/vad/silero.rb +344 -0
  76. data/lib/cohere/transcribe/vad/silero_vad_v6.onnx +0 -0
  77. data/lib/cohere/transcribe/vad/timestamps.rb +219 -0
  78. data/lib/cohere/transcribe/version.rb +7 -0
  79. data/lib/cohere/transcribe.rb +26 -0
  80. data/sig/cohere/transcribe.rbs +250 -0
  81. data/vendor/crispasr/AUTHORS +510 -0
  82. data/vendor/crispasr/LICENSE +21 -0
  83. data/vendor/crispasr/UPSTREAM.md +9 -0
  84. data/vendor/crispasr/VERSION +1 -0
  85. data/vendor/crispasr/ggml/AUTHORS +335 -0
  86. data/vendor/crispasr/ggml/CMakeLists.txt +512 -0
  87. data/vendor/crispasr/ggml/LICENSE +21 -0
  88. data/vendor/crispasr/ggml/README.md +49 -0
  89. data/vendor/crispasr/ggml/cmake/FindNCCL.cmake +36 -0
  90. data/vendor/crispasr/ggml/cmake/GitVars.cmake +22 -0
  91. data/vendor/crispasr/ggml/cmake/common.cmake +50 -0
  92. data/vendor/crispasr/ggml/cmake/ggml-config.cmake.in +191 -0
  93. data/vendor/crispasr/ggml/ggml.pc.in +10 -0
  94. data/vendor/crispasr/ggml/include/ggml-alloc.h +85 -0
  95. data/vendor/crispasr/ggml/include/ggml-backend.h +431 -0
  96. data/vendor/crispasr/ggml/include/ggml-blas.h +25 -0
  97. data/vendor/crispasr/ggml/include/ggml-cann.h +123 -0
  98. data/vendor/crispasr/ggml/include/ggml-cpp.h +39 -0
  99. data/vendor/crispasr/ggml/include/ggml-cpu.h +151 -0
  100. data/vendor/crispasr/ggml/include/ggml-cuda.h +50 -0
  101. data/vendor/crispasr/ggml/include/ggml-hexagon.h +19 -0
  102. data/vendor/crispasr/ggml/include/ggml-metal.h +61 -0
  103. data/vendor/crispasr/ggml/include/ggml-opencl.h +26 -0
  104. data/vendor/crispasr/ggml/include/ggml-openvino.h +37 -0
  105. data/vendor/crispasr/ggml/include/ggml-opt.h +256 -0
  106. data/vendor/crispasr/ggml/include/ggml-rpc.h +35 -0
  107. data/vendor/crispasr/ggml/include/ggml-sycl.h +49 -0
  108. data/vendor/crispasr/ggml/include/ggml-virtgpu.h +14 -0
  109. data/vendor/crispasr/ggml/include/ggml-vulkan.h +29 -0
  110. data/vendor/crispasr/ggml/include/ggml-webgpu.h +19 -0
  111. data/vendor/crispasr/ggml/include/ggml-zdnn.h +17 -0
  112. data/vendor/crispasr/ggml/include/ggml-zendnn.h +22 -0
  113. data/vendor/crispasr/ggml/include/ggml.h +2887 -0
  114. data/vendor/crispasr/ggml/include/gguf.h +204 -0
  115. data/vendor/crispasr/ggml/src/CMakeLists.txt +493 -0
  116. data/vendor/crispasr/ggml/src/ggml-alloc.c +1323 -0
  117. data/vendor/crispasr/ggml/src/ggml-backend-dl.cpp +48 -0
  118. data/vendor/crispasr/ggml/src/ggml-backend-dl.h +44 -0
  119. data/vendor/crispasr/ggml/src/ggml-backend-impl.h +275 -0
  120. data/vendor/crispasr/ggml/src/ggml-backend-meta.cpp +2145 -0
  121. data/vendor/crispasr/ggml/src/ggml-backend-reg.cpp +586 -0
  122. data/vendor/crispasr/ggml/src/ggml-backend.cpp +2437 -0
  123. data/vendor/crispasr/ggml/src/ggml-common.h +1900 -0
  124. data/vendor/crispasr/ggml/src/ggml-cpu/CMakeLists.txt +718 -0
  125. data/vendor/crispasr/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  126. data/vendor/crispasr/ggml/src/ggml-cpu/amx/amx.h +8 -0
  127. data/vendor/crispasr/ggml/src/ggml-cpu/amx/common.h +115 -0
  128. data/vendor/crispasr/ggml/src/ggml-cpu/amx/mmq.cpp +2512 -0
  129. data/vendor/crispasr/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  130. data/vendor/crispasr/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +98 -0
  131. data/vendor/crispasr/ggml/src/ggml-cpu/arch/arm/quants.c +4244 -0
  132. data/vendor/crispasr/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  133. data/vendor/crispasr/ggml/src/ggml-cpu/arch/loongarch/quants.c +2158 -0
  134. data/vendor/crispasr/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  135. data/vendor/crispasr/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  136. data/vendor/crispasr/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  137. data/vendor/crispasr/ggml/src/ggml-cpu/arch/riscv/quants.c +4455 -0
  138. data/vendor/crispasr/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  139. data/vendor/crispasr/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  140. data/vendor/crispasr/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  141. data/vendor/crispasr/ggml/src/ggml-cpu/arch/wasm/quants.c +1220 -0
  142. data/vendor/crispasr/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  143. data/vendor/crispasr/ggml/src/ggml-cpu/arch/x86/quants.c +3970 -0
  144. data/vendor/crispasr/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  145. data/vendor/crispasr/ggml/src/ggml-cpu/arch-fallback.h +349 -0
  146. data/vendor/crispasr/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  147. data/vendor/crispasr/ggml/src/ggml-cpu/binary-ops.h +16 -0
  148. data/vendor/crispasr/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  149. data/vendor/crispasr/ggml/src/ggml-cpu/common.h +95 -0
  150. data/vendor/crispasr/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  151. data/vendor/crispasr/ggml/src/ggml-cpu/ggml-cpu.c +3791 -0
  152. data/vendor/crispasr/ggml/src/ggml-cpu/ggml-cpu.cpp +703 -0
  153. data/vendor/crispasr/ggml/src/ggml-cpu/hbm.cpp +55 -0
  154. data/vendor/crispasr/ggml/src/ggml-cpu/hbm.h +8 -0
  155. data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kernels.cpp +939 -0
  156. data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kernels.h +90 -0
  157. data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1513 -0
  158. data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  159. data/vendor/crispasr/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4051 -0
  160. data/vendor/crispasr/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  161. data/vendor/crispasr/ggml/src/ggml-cpu/ops.cpp +11662 -0
  162. data/vendor/crispasr/ggml/src/ggml-cpu/ops.h +121 -0
  163. data/vendor/crispasr/ggml/src/ggml-cpu/quants.c +1288 -0
  164. data/vendor/crispasr/ggml/src/ggml-cpu/quants.h +103 -0
  165. data/vendor/crispasr/ggml/src/ggml-cpu/repack.cpp +4836 -0
  166. data/vendor/crispasr/ggml/src/ggml-cpu/repack.h +245 -0
  167. data/vendor/crispasr/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  168. data/vendor/crispasr/ggml/src/ggml-cpu/simd-mappings.h +1329 -0
  169. data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime.cpp +1025 -0
  170. data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime.h +13 -0
  171. data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +3196 -0
  172. data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime_kernels.h +26 -0
  173. data/vendor/crispasr/ggml/src/ggml-cpu/traits.cpp +36 -0
  174. data/vendor/crispasr/ggml/src/ggml-cpu/traits.h +38 -0
  175. data/vendor/crispasr/ggml/src/ggml-cpu/unary-ops.cpp +336 -0
  176. data/vendor/crispasr/ggml/src/ggml-cpu/unary-ops.h +35 -0
  177. data/vendor/crispasr/ggml/src/ggml-cpu/vec.cpp +681 -0
  178. data/vendor/crispasr/ggml/src/ggml-cpu/vec.h +1606 -0
  179. data/vendor/crispasr/ggml/src/ggml-cuda/CMakeLists.txt +272 -0
  180. data/vendor/crispasr/ggml/src/ggml-cuda/acc.cu +61 -0
  181. data/vendor/crispasr/ggml/src/ggml-cuda/acc.cuh +5 -0
  182. data/vendor/crispasr/ggml/src/ggml-cuda/add-id.cu +58 -0
  183. data/vendor/crispasr/ggml/src/ggml-cuda/add-id.cuh +3 -0
  184. data/vendor/crispasr/ggml/src/ggml-cuda/arange.cu +34 -0
  185. data/vendor/crispasr/ggml/src/ggml-cuda/arange.cuh +5 -0
  186. data/vendor/crispasr/ggml/src/ggml-cuda/argmax.cu +91 -0
  187. data/vendor/crispasr/ggml/src/ggml-cuda/argmax.cuh +3 -0
  188. data/vendor/crispasr/ggml/src/ggml-cuda/argsort.cu +265 -0
  189. data/vendor/crispasr/ggml/src/ggml-cuda/argsort.cuh +19 -0
  190. data/vendor/crispasr/ggml/src/ggml-cuda/binbcast.cu +534 -0
  191. data/vendor/crispasr/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  192. data/vendor/crispasr/ggml/src/ggml-cuda/clamp.cu +45 -0
  193. data/vendor/crispasr/ggml/src/ggml-cuda/clamp.cuh +5 -0
  194. data/vendor/crispasr/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  195. data/vendor/crispasr/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  196. data/vendor/crispasr/ggml/src/ggml-cuda/common.cuh +1489 -0
  197. data/vendor/crispasr/ggml/src/ggml-cuda/concat.cu +204 -0
  198. data/vendor/crispasr/ggml/src/ggml-cuda/concat.cuh +5 -0
  199. data/vendor/crispasr/ggml/src/ggml-cuda/conv-transpose-1d.cu +97 -0
  200. data/vendor/crispasr/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  201. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  202. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  203. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  204. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  205. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d.cu +166 -0
  206. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  207. data/vendor/crispasr/ggml/src/ggml-cuda/convert.cu +892 -0
  208. data/vendor/crispasr/ggml/src/ggml-cuda/convert.cuh +66 -0
  209. data/vendor/crispasr/ggml/src/ggml-cuda/count-equal.cu +64 -0
  210. data/vendor/crispasr/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  211. data/vendor/crispasr/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  212. data/vendor/crispasr/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  213. data/vendor/crispasr/ggml/src/ggml-cuda/cpy.cu +581 -0
  214. data/vendor/crispasr/ggml/src/ggml-cuda/cpy.cuh +7 -0
  215. data/vendor/crispasr/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  216. data/vendor/crispasr/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  217. data/vendor/crispasr/ggml/src/ggml-cuda/cumsum.cu +307 -0
  218. data/vendor/crispasr/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  219. data/vendor/crispasr/ggml/src/ggml-cuda/dequantize.cuh +99 -0
  220. data/vendor/crispasr/ggml/src/ggml-cuda/diag.cu +77 -0
  221. data/vendor/crispasr/ggml/src/ggml-cuda/diag.cuh +5 -0
  222. data/vendor/crispasr/ggml/src/ggml-cuda/diagmask.cu +40 -0
  223. data/vendor/crispasr/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  224. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-common.cuh +1212 -0
  225. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1860 -0
  226. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-tile.cu +57 -0
  227. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-tile.cuh +1309 -0
  228. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-vec.cuh +600 -0
  229. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-wmma-f16.cu +696 -0
  230. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +51 -0
  231. data/vendor/crispasr/ggml/src/ggml-cuda/fattn.cu +620 -0
  232. data/vendor/crispasr/ggml/src/ggml-cuda/fattn.cuh +5 -0
  233. data/vendor/crispasr/ggml/src/ggml-cuda/fill.cu +37 -0
  234. data/vendor/crispasr/ggml/src/ggml-cuda/fill.cuh +3 -0
  235. data/vendor/crispasr/ggml/src/ggml-cuda/gated_delta_net.cu +273 -0
  236. data/vendor/crispasr/ggml/src/ggml-cuda/gated_delta_net.cuh +4 -0
  237. data/vendor/crispasr/ggml/src/ggml-cuda/getrows.cu +332 -0
  238. data/vendor/crispasr/ggml/src/ggml-cuda/getrows.cuh +15 -0
  239. data/vendor/crispasr/ggml/src/ggml-cuda/ggml-cuda.cu +5580 -0
  240. data/vendor/crispasr/ggml/src/ggml-cuda/gla.cu +93 -0
  241. data/vendor/crispasr/ggml/src/ggml-cuda/gla.cuh +3 -0
  242. data/vendor/crispasr/ggml/src/ggml-cuda/im2col.cu +274 -0
  243. data/vendor/crispasr/ggml/src/ggml-cuda/im2col.cuh +6 -0
  244. data/vendor/crispasr/ggml/src/ggml-cuda/mean.cu +75 -0
  245. data/vendor/crispasr/ggml/src/ggml-cuda/mean.cuh +3 -0
  246. data/vendor/crispasr/ggml/src/ggml-cuda/mma.cuh +1333 -0
  247. data/vendor/crispasr/ggml/src/ggml-cuda/mmf.cu +191 -0
  248. data/vendor/crispasr/ggml/src/ggml-cuda/mmf.cuh +908 -0
  249. data/vendor/crispasr/ggml/src/ggml-cuda/mmid.cu +164 -0
  250. data/vendor/crispasr/ggml/src/ggml-cuda/mmid.cuh +5 -0
  251. data/vendor/crispasr/ggml/src/ggml-cuda/mmq.cu +372 -0
  252. data/vendor/crispasr/ggml/src/ggml-cuda/mmq.cuh +4175 -0
  253. data/vendor/crispasr/ggml/src/ggml-cuda/mmvf.cu +862 -0
  254. data/vendor/crispasr/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  255. data/vendor/crispasr/ggml/src/ggml-cuda/mmvq.cu +1161 -0
  256. data/vendor/crispasr/ggml/src/ggml-cuda/mmvq.cuh +16 -0
  257. data/vendor/crispasr/ggml/src/ggml-cuda/norm.cu +756 -0
  258. data/vendor/crispasr/ggml/src/ggml-cuda/norm.cuh +20 -0
  259. data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  260. data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  261. data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  262. data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  263. data/vendor/crispasr/ggml/src/ggml-cuda/out-prod.cu +68 -0
  264. data/vendor/crispasr/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  265. data/vendor/crispasr/ggml/src/ggml-cuda/pad.cu +106 -0
  266. data/vendor/crispasr/ggml/src/ggml-cuda/pad.cuh +5 -0
  267. data/vendor/crispasr/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  268. data/vendor/crispasr/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  269. data/vendor/crispasr/ggml/src/ggml-cuda/pool2d.cu +94 -0
  270. data/vendor/crispasr/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  271. data/vendor/crispasr/ggml/src/ggml-cuda/quantize.cu +443 -0
  272. data/vendor/crispasr/ggml/src/ggml-cuda/quantize.cuh +41 -0
  273. data/vendor/crispasr/ggml/src/ggml-cuda/reduce_rows.cuh +39 -0
  274. data/vendor/crispasr/ggml/src/ggml-cuda/roll.cu +67 -0
  275. data/vendor/crispasr/ggml/src/ggml-cuda/roll.cuh +5 -0
  276. data/vendor/crispasr/ggml/src/ggml-cuda/rope.cu +665 -0
  277. data/vendor/crispasr/ggml/src/ggml-cuda/rope.cuh +9 -0
  278. data/vendor/crispasr/ggml/src/ggml-cuda/scale.cu +34 -0
  279. data/vendor/crispasr/ggml/src/ggml-cuda/scale.cuh +5 -0
  280. data/vendor/crispasr/ggml/src/ggml-cuda/set-rows.cu +330 -0
  281. data/vendor/crispasr/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  282. data/vendor/crispasr/ggml/src/ggml-cuda/set.cu +39 -0
  283. data/vendor/crispasr/ggml/src/ggml-cuda/set.cuh +7 -0
  284. data/vendor/crispasr/ggml/src/ggml-cuda/softcap.cu +34 -0
  285. data/vendor/crispasr/ggml/src/ggml-cuda/softcap.cuh +5 -0
  286. data/vendor/crispasr/ggml/src/ggml-cuda/softmax.cu +472 -0
  287. data/vendor/crispasr/ggml/src/ggml-cuda/softmax.cuh +7 -0
  288. data/vendor/crispasr/ggml/src/ggml-cuda/solve_tri.cu +275 -0
  289. data/vendor/crispasr/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  290. data/vendor/crispasr/ggml/src/ggml-cuda/ssm-conv.cu +197 -0
  291. data/vendor/crispasr/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  292. data/vendor/crispasr/ggml/src/ggml-cuda/ssm-scan.cu +342 -0
  293. data/vendor/crispasr/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  294. data/vendor/crispasr/ggml/src/ggml-cuda/sum.cu +41 -0
  295. data/vendor/crispasr/ggml/src/ggml-cuda/sum.cuh +5 -0
  296. data/vendor/crispasr/ggml/src/ggml-cuda/sumrows.cu +43 -0
  297. data/vendor/crispasr/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  298. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
  299. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  300. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +11 -0
  301. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  302. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
  303. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  304. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
  305. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  306. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  307. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +11 -0
  308. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  309. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
  310. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
  311. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
  312. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  313. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +11 -0
  314. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  315. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  316. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
  317. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  318. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +11 -0
  319. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  320. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  321. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  322. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  323. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  324. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  325. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  326. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  327. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  328. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  329. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  330. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  331. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  332. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  333. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  334. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  335. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  336. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  337. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  338. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  339. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  340. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  341. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  342. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  343. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  344. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  345. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  346. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  347. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  348. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  349. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  350. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  351. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  352. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  353. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  354. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  355. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  356. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  357. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  358. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  359. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  360. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  361. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  362. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  363. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  364. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  365. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  366. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  367. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  368. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  369. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  370. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  371. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  372. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  373. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  374. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  375. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  376. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  377. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  378. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  379. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  380. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  381. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  382. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  383. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  384. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  385. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  386. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  387. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  388. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  389. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  390. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  391. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  392. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  393. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  394. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  395. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  396. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  397. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  398. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  399. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  400. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  401. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  402. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  403. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  404. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  405. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  406. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  407. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  408. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  409. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  410. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  411. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  412. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  413. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  414. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  415. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  416. data/vendor/crispasr/ggml/src/ggml-cuda/top-k.cu +95 -0
  417. data/vendor/crispasr/ggml/src/ggml-cuda/top-k.cuh +3 -0
  418. data/vendor/crispasr/ggml/src/ggml-cuda/topk-moe.cu +415 -0
  419. data/vendor/crispasr/ggml/src/ggml-cuda/topk-moe.cuh +27 -0
  420. data/vendor/crispasr/ggml/src/ggml-cuda/tri.cu +136 -0
  421. data/vendor/crispasr/ggml/src/ggml-cuda/tri.cuh +5 -0
  422. data/vendor/crispasr/ggml/src/ggml-cuda/tsembd.cu +47 -0
  423. data/vendor/crispasr/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  424. data/vendor/crispasr/ggml/src/ggml-cuda/unary.cu +644 -0
  425. data/vendor/crispasr/ggml/src/ggml-cuda/unary.cuh +116 -0
  426. data/vendor/crispasr/ggml/src/ggml-cuda/upscale.cu +293 -0
  427. data/vendor/crispasr/ggml/src/ggml-cuda/upscale.cuh +5 -0
  428. data/vendor/crispasr/ggml/src/ggml-cuda/vecdotq.cuh +1317 -0
  429. data/vendor/crispasr/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  430. data/vendor/crispasr/ggml/src/ggml-cuda/vendors/hip.h +306 -0
  431. data/vendor/crispasr/ggml/src/ggml-cuda/vendors/musa.h +148 -0
  432. data/vendor/crispasr/ggml/src/ggml-cuda/wkv.cu +199 -0
  433. data/vendor/crispasr/ggml/src/ggml-cuda/wkv.cuh +7 -0
  434. data/vendor/crispasr/ggml/src/ggml-impl.h +783 -0
  435. data/vendor/crispasr/ggml/src/ggml-metal/CMakeLists.txt +124 -0
  436. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-common.cpp +457 -0
  437. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-common.h +52 -0
  438. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  439. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-context.m +778 -0
  440. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-device.cpp +2117 -0
  441. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-device.h +299 -0
  442. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-device.m +2093 -0
  443. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-impl.h +1267 -0
  444. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-ops.cpp +5023 -0
  445. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-ops.h +111 -0
  446. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal.cpp +954 -0
  447. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal.metal +11756 -0
  448. data/vendor/crispasr/ggml/src/ggml-opt.cpp +1094 -0
  449. data/vendor/crispasr/ggml/src/ggml-quants.c +5491 -0
  450. data/vendor/crispasr/ggml/src/ggml-quants.h +112 -0
  451. data/vendor/crispasr/ggml/src/ggml-threading.cpp +12 -0
  452. data/vendor/crispasr/ggml/src/ggml-threading.h +14 -0
  453. data/vendor/crispasr/ggml/src/ggml.c +7925 -0
  454. data/vendor/crispasr/ggml/src/ggml.cpp +26 -0
  455. data/vendor/crispasr/ggml/src/gguf.cpp +1556 -0
  456. data/vendor/crispasr/src/cohere-arch.h +137 -0
  457. data/vendor/crispasr/src/cohere.cpp +5642 -0
  458. data/vendor/crispasr/src/cohere.h +327 -0
  459. data/vendor/crispasr/src/cohere_batch_planner.h +82 -0
  460. data/vendor/crispasr/src/cohere_chunking.h +64 -0
  461. data/vendor/crispasr/src/cohere_decoder_batch_layout.h +60 -0
  462. data/vendor/crispasr/src/cohere_encoder_padded_layout.h +27 -0
  463. data/vendor/crispasr/src/cohere_frontend.cpp +189 -0
  464. data/vendor/crispasr/src/cohere_frontend.h +31 -0
  465. data/vendor/crispasr/src/cohere_ragged_controller.h +149 -0
  466. data/vendor/crispasr/src/cohere_token_renderer.h +181 -0
  467. data/vendor/crispasr/src/core/attention.h +924 -0
  468. data/vendor/crispasr/src/core/audio_chunking.h +97 -0
  469. data/vendor/crispasr/src/core/beam_decode.h +486 -0
  470. data/vendor/crispasr/src/core/cpu_ops.h +135 -0
  471. data/vendor/crispasr/src/core/gguf_loader.cpp +1021 -0
  472. data/vendor/crispasr/src/core/gguf_loader.h +216 -0
  473. data/vendor/crispasr/src/core/gpu_backend_pref.h +119 -0
  474. data/vendor/crispasr/src/core/mel.cpp +519 -0
  475. data/vendor/crispasr/src/core/mel.h +265 -0
  476. data/vendor/crispasr/src/core/ngram_loop_fix.h +173 -0
  477. data/vendor/crispasr/src/core/repetition_loop_guard.h +54 -0
  478. data/vendor/crispasr/src/crispasr_imatrix.cpp +255 -0
  479. data/vendor/crispasr/src/crispasr_imatrix.h +38 -0
  480. metadata +596 -0
@@ -0,0 +1,97 @@
1
+ // src/core/audio_chunking.h — boundary detection for long-form audio.
2
+ //
3
+ // Backends with bounded encoder windows (Cohere model rows are <= 35 s, parakeet/canary
4
+ // can also benefit) currently cut at exactly N * sample_rate samples,
5
+ // which slices mid-word and corrupts the transcript at chunk seams.
6
+ //
7
+ // `find_energy_min_split` scans a 1-D mono PCM segment in fixed-size
8
+ // non-overlapping windows and returns the start index of the lowest-RMS
9
+ // window — i.e. the quietest 100 ms within a search range. Cutting
10
+ // there avoids splitting a syllable in two and keeps the encoder /
11
+ // decoder operating on coherent acoustic boundaries.
12
+ //
13
+ // `split_at_energy_minima` wraps that into a chunker that yields
14
+ // [begin, end) sample ranges of length <= max_chunk_samples, choosing
15
+ // each cut from the last `boundary_context_samples` of the running
16
+ // window so we land in a quiet point near the cap. Mirrors the
17
+ // `_find_split_point_energy` / `split_audio_chunks_energy` helpers in
18
+ // nano-cohere-transcribe (Apache 2.0); ported to C++ for use across
19
+ // CrispASR's encoder backends.
20
+ #pragma once
21
+
22
+ #include <cmath>
23
+ #include <cstddef>
24
+ #include <utility>
25
+ #include <vector>
26
+
27
+ namespace audio_chunking {
28
+
29
+ // Return the start index (relative to `samples[search_start]`'s base
30
+ // position in the caller's frame) of the lowest-RMS non-overlapping
31
+ // window of length `win_samples` inside [search_start, search_end).
32
+ //
33
+ // `search_end - search_start <= win_samples` is treated as a degenerate
34
+ // case and returns the midpoint of the search range.
35
+ inline size_t find_energy_min_split(const float* samples, size_t search_start, size_t search_end, size_t win_samples) {
36
+ if (search_end <= search_start)
37
+ return search_start;
38
+ const size_t span = search_end - search_start;
39
+ if (span <= win_samples)
40
+ return search_start + span / 2;
41
+ double min_e = std::numeric_limits<double>::infinity();
42
+ size_t best = search_start;
43
+ // Match the Cohere reference's `range(0, span - win, win)`: the final
44
+ // window whose end is exactly search_end is deliberately not a candidate.
45
+ for (size_t i = 0; i + win_samples < span; i += win_samples) {
46
+ double s = 0.0;
47
+ for (size_t j = 0; j < win_samples; ++j) {
48
+ const float v = samples[search_start + i + j];
49
+ s += (double)v * (double)v;
50
+ }
51
+ const double e = std::sqrt(s / (double)win_samples);
52
+ if (e < min_e) {
53
+ min_e = e;
54
+ best = search_start + i;
55
+ }
56
+ }
57
+ return best;
58
+ }
59
+
60
+ // Split `samples[0 .. n_samples)` into <= max_chunk_samples chunks,
61
+ // preferring cuts inside the last `search_window_samples` of each
62
+ // running window where RMS is lowest. Returns [begin, end) ranges in
63
+ // sample units. If the input is shorter than max_chunk_samples, returns
64
+ // a single [0, n_samples) range.
65
+ inline std::vector<std::pair<size_t, size_t>> split_at_energy_minima(const float* samples, size_t n_samples,
66
+ size_t max_chunk_samples,
67
+ size_t search_window_samples,
68
+ size_t win_samples = 1600) {
69
+ std::vector<std::pair<size_t, size_t>> out;
70
+ if (n_samples == 0)
71
+ return out;
72
+ if (max_chunk_samples == 0 || n_samples <= max_chunk_samples) {
73
+ out.emplace_back(0, n_samples);
74
+ return out;
75
+ }
76
+ size_t idx = 0;
77
+ while (idx < n_samples) {
78
+ if (idx + max_chunk_samples >= n_samples) {
79
+ out.emplace_back(idx, n_samples);
80
+ break;
81
+ }
82
+ const size_t search_start =
83
+ (search_window_samples >= max_chunk_samples) ? idx : idx + max_chunk_samples - search_window_samples;
84
+ const size_t search_end = idx + max_chunk_samples;
85
+ size_t cut = find_energy_min_split(samples, search_start, search_end, win_samples);
86
+ // Defensive clamp: forward progress, never past the cap.
87
+ if (cut <= idx)
88
+ cut = idx + max_chunk_samples;
89
+ if (cut > n_samples)
90
+ cut = n_samples;
91
+ out.emplace_back(idx, cut);
92
+ idx = cut;
93
+ }
94
+ return out;
95
+ }
96
+
97
+ } // namespace audio_chunking
@@ -0,0 +1,486 @@
1
+ // src/core/beam_decode.h — shared autoregressive beam-search decode loop.
2
+ //
3
+ // Companion to `core_greedy_decode::run_with_probs`. Same callback shape
4
+ // at the call site (a single `replay_fn`), but instead of picking one
5
+ // argmax / sampled token per step, this helper expands the top
6
+ // `beam_size` hypotheses in parallel and prunes globally to keep the
7
+ // highest-cumulative-logprob beams alive. Returns the winning beam's
8
+ // tokens + per-token softmax probabilities.
9
+ //
10
+ // KV strategy — replay-from-prefix
11
+ // --------------------------------
12
+ // Beam search needs each beam to have its own KV state, but the LLM-style
13
+ // runtime backends (glm-asr, kyutai-stt, omniasr-llm, moonshine via
14
+ // decode step) keep KV in a single context-owned buffer. To avoid adding
15
+ // a per-backend `kv_save` / `kv_restore` API, we exploit the fact that
16
+ // every call to "advance the LM by these tokens at this n_past" lets us
17
+ // write KV slots at any logical position. Each step rebuilds each beam's
18
+ // KV by replaying its full generated suffix from the post-prompt anchor.
19
+ //
20
+ // Cost: O(beam_size × T²/2) extra forward work for T generated tokens vs
21
+ // greedy's O(T). Acceptable for beam_size = 2-4 since the audio encoder
22
+ // typically dominates wall time on these backends. If perf is bad on
23
+ // long generations, the right next step is `*_kv_save` / `*_kv_restore`
24
+ // per backend, not making this helper smarter.
25
+ //
26
+ // Caller contract
27
+ // ---------------
28
+ // Caller is responsible for:
29
+ // * Running the prompt prefill so KV slots [0, prompt_len) hold the
30
+ // prompt's K/V.
31
+ // * Capturing the prefill logits at the last prompt position.
32
+ // * Providing a `replay_fn(ctx, tokens, n_tokens, prompt_len)` that
33
+ // overwrites KV slots [prompt_len, prompt_len + n_tokens) with the
34
+ // given suffix's K/V and returns the last-position logits as a
35
+ // malloc'd `float*` of size `vocab_size` (or nullptr on failure).
36
+ // The helper free()s the returned buffer.
37
+ // * Resetting KV after `run_with_probs` returns (state is undefined
38
+ // since each beam-step overwrites slots).
39
+ //
40
+ // For backends that natively expose a batched
41
+ // `forward(ctx, embeds, n_tokens, n_past)` — like glm-asr's
42
+ // `glm_asr_run_llm_kv` — the replay_fn is a one-liner that embeds +
43
+ // forwards. For backends with a per-token API (like omniasr-llm's
44
+ // `omniasr_run_dec_token`), the replay_fn loops over tokens internally.
45
+ //
46
+ // Usage (glm-asr):
47
+ //
48
+ // core_beam_decode::Config cfg;
49
+ // cfg.max_new_tokens = 512;
50
+ // cfg.eos_id = hp.eos_token_ids[0];
51
+ // cfg.vocab_size = hp.llm_vocab;
52
+ // cfg.beam_size = params.beam_size;
53
+ // cfg.prompt_len = (int)prompt_ids.size();
54
+ //
55
+ // auto replay = [ctx](const int32_t* toks, int n, int prompt_len) -> float* {
56
+ // float* emb = glm_asr_embed_tokens(ctx, toks, n);
57
+ // if (!emb) return nullptr;
58
+ // float* lg = glm_asr_run_llm_kv(ctx, emb, n, prompt_len, nullptr, nullptr);
59
+ // std::free(emb);
60
+ // return lg;
61
+ // };
62
+ //
63
+ // auto r = core_beam_decode::run_with_probs(ctx, prefill_lg, replay, cfg);
64
+ //
65
+ // Header-only so the compiler inlines each caller's concrete callable
66
+ // at the call site (matching the greedy helper's pattern).
67
+
68
+ #pragma once
69
+
70
+ #include <algorithm>
71
+ #include <cmath>
72
+ #include <cstdint>
73
+ #include <cstdlib>
74
+ #include <memory>
75
+ #include <type_traits>
76
+ #include <utility>
77
+ #include <vector>
78
+
79
+ namespace core_beam_decode {
80
+
81
+ struct Config {
82
+ int max_new_tokens = 512; // hard cap on generated tokens
83
+ int eos_id = 2; // legacy single-EOS field; used iff eos_ids is empty
84
+ std::vector<int> eos_ids; // multi-EOS set (e.g. glm-asr's 3 stop tokens). Any match terminates the beam.
85
+ int vocab_size = 0; // required
86
+ int beam_size = 1; // 1 = degenerate to greedy-via-beam (still works, just expensive)
87
+ int prompt_len = 0; // n_past after prompt prefill (replay anchor)
88
+ };
89
+
90
+ struct Result {
91
+ std::vector<int32_t> tokens; // generated tokens of the winning beam
92
+ std::vector<float> probs; // softmax prob of each picked token in [0,1]
93
+ };
94
+
95
+ namespace detail {
96
+
97
+ // Internal beam state. Tokens are the generated suffix only (post-prompt).
98
+ struct Beam {
99
+ std::vector<int32_t> tokens;
100
+ std::vector<float> probs;
101
+ double cum_logprob = 0.0;
102
+ bool finished = false;
103
+ };
104
+
105
+ // Numerically-stable log-Z (log normaliser of softmax over `logits`).
106
+ inline double compute_logZ(const float* logits, int vocab) {
107
+ float mx = logits[0];
108
+ for (int k = 1; k < vocab; k++)
109
+ if (logits[k] > mx)
110
+ mx = logits[k];
111
+ double sum = 0.0;
112
+ for (int k = 0; k < vocab; k++)
113
+ sum += std::exp((double)(logits[k] - mx));
114
+ return (double)mx + std::log(sum);
115
+ }
116
+
117
+ // Pick top-K tokens from `logits` and return their log-softmax values.
118
+ // `out_ids` and `out_lps` are sized to K, sorted descending by logit.
119
+ //
120
+ // Uses a min-heap of size K: O(V log K) instead of the previous
121
+ // O(V + V log K) partial_sort that allocated a full vocab-sized index
122
+ // vector per call (§176r). For beam_size=4 and vocab=32K-150K this
123
+ // eliminates a ~128-600 KB alloc per beam expansion step.
124
+ inline void top_k_log_softmax(const float* logits, int vocab, int K, std::vector<int>& out_ids,
125
+ std::vector<double>& out_lps) {
126
+ if (K > vocab)
127
+ K = vocab;
128
+ out_ids.resize((size_t)K);
129
+ out_lps.resize((size_t)K);
130
+
131
+ const double logZ = compute_logZ(logits, vocab);
132
+
133
+ // Min-heap: stores (logit, token_id) pairs; the smallest logit is
134
+ // at the top so we can cheaply evict it when a larger one arrives.
135
+ // Reused across calls (thread-local) so the K-element heap isn't
136
+ // re-allocated on every beam-expansion step (B×T per transcribe). clear()
137
+ // keeps capacity; the heap is rebuilt from scratch below, so the result is
138
+ // bit-identical to a fresh vector. (§176r follow-up)
139
+ using Pair = std::pair<float, int>;
140
+ static thread_local std::vector<Pair> heap;
141
+ heap.clear();
142
+ heap.reserve((size_t)K);
143
+ for (int i = 0; i < vocab; i++) {
144
+ if ((int)heap.size() < K) {
145
+ heap.push_back({logits[i], i});
146
+ if ((int)heap.size() == K)
147
+ std::make_heap(heap.begin(), heap.end(),
148
+ [](const Pair& a, const Pair& b) { return a.first > b.first; });
149
+ } else if (logits[i] > heap[0].first) {
150
+ std::pop_heap(heap.begin(), heap.end(), [](const Pair& a, const Pair& b) { return a.first > b.first; });
151
+ heap.back() = {logits[i], i};
152
+ std::push_heap(heap.begin(), heap.end(), [](const Pair& a, const Pair& b) { return a.first > b.first; });
153
+ }
154
+ }
155
+ // Sort descending by logit.
156
+ std::sort(heap.begin(), heap.end(), [](const Pair& a, const Pair& b) { return a.first > b.first; });
157
+ for (int i = 0; i < K; i++) {
158
+ out_ids[(size_t)i] = heap[(size_t)i].second;
159
+ out_lps[(size_t)i] = (double)heap[(size_t)i].first - logZ;
160
+ }
161
+ }
162
+
163
+ } // namespace detail
164
+
165
+ // Run the beam decode loop.
166
+ //
167
+ // Template parameters:
168
+ // Ctx : the model's opaque context type.
169
+ // ReplayFn : callable with signature
170
+ // `float * (Ctx*, const int32_t* tokens, int n_tokens, int prompt_len)`.
171
+ // Must overwrite KV slots [prompt_len, prompt_len + n_tokens)
172
+ // and return a malloc'd `float*` containing the last-position
173
+ // logits ([vocab_size, 1]). Returning nullptr terminates that
174
+ // beam.
175
+ //
176
+ // Pre-condition: caller has populated KV slots [0, prompt_len) with the
177
+ // prompt's K/V; `prefill_logits` is the [V, 1] logits at the last prompt
178
+ // position (caller-owned, not freed by this helper).
179
+ //
180
+ // Post-condition: KV is left in an undefined state — caller must reset
181
+ // it before the next transcription.
182
+ template <typename Ctx, typename ReplayFn>
183
+ inline Result run_with_probs(Ctx* ctx, const float* prefill_logits, ReplayFn replay_fn, const Config& cfg) {
184
+ using detail::Beam;
185
+
186
+ Result result;
187
+ if (!prefill_logits || cfg.vocab_size <= 0)
188
+ return result;
189
+
190
+ const int B = (cfg.beam_size > 0) ? cfg.beam_size : 1;
191
+ const int V = cfg.vocab_size;
192
+
193
+ auto is_eos = [&](int id) {
194
+ if (!cfg.eos_ids.empty()) {
195
+ for (int e : cfg.eos_ids)
196
+ if (id == e)
197
+ return true;
198
+ return false;
199
+ }
200
+ return id == cfg.eos_id;
201
+ };
202
+
203
+ // 1. Initial beams: top-B from prefill logits.
204
+ std::vector<int> first_ids;
205
+ std::vector<double> first_lps;
206
+ detail::top_k_log_softmax(prefill_logits, V, B, first_ids, first_lps);
207
+
208
+ std::vector<Beam> beams((size_t)first_ids.size());
209
+ for (size_t i = 0; i < first_ids.size(); i++) {
210
+ beams[i].tokens.push_back(first_ids[i]);
211
+ beams[i].probs.push_back((float)std::exp(first_lps[i]));
212
+ beams[i].cum_logprob = first_lps[i];
213
+ if (is_eos(first_ids[i]))
214
+ beams[i].finished = true;
215
+ }
216
+ // top_k_log_softmax already returns descending; beams are sorted.
217
+
218
+ if (beams.empty())
219
+ return result;
220
+
221
+ // 2. Per-step expand-and-prune.
222
+ while ((int)beams[0].tokens.size() < cfg.max_new_tokens && !beams[0].finished) {
223
+ struct Cand {
224
+ int beam_idx;
225
+ int token;
226
+ double cum_logprob;
227
+ float token_prob;
228
+ bool from_finished; // carry-forward of an already-finished beam
229
+ };
230
+ std::vector<Cand> cands;
231
+ cands.reserve((size_t)B * (size_t)B + (size_t)B);
232
+
233
+ for (size_t bi = 0; bi < beams.size(); bi++) {
234
+ auto& b = beams[bi];
235
+ if (b.finished) {
236
+ // Carry forward; the token field is only used by the
237
+ // detokeniser, which skips finished beams.
238
+ const int sentinel = cfg.eos_ids.empty() ? cfg.eos_id : cfg.eos_ids[0];
239
+ cands.push_back({(int)bi, sentinel, b.cum_logprob, 1.0f, true});
240
+ continue;
241
+ }
242
+
243
+ float* lg = replay_fn(ctx, b.tokens.data(), (int)b.tokens.size(), cfg.prompt_len);
244
+ if (!lg) {
245
+ b.finished = true;
246
+ continue;
247
+ }
248
+
249
+ std::vector<int> ids;
250
+ std::vector<double> lps;
251
+ detail::top_k_log_softmax(lg, V, B, ids, lps);
252
+ std::free(lg);
253
+
254
+ for (size_t j = 0; j < ids.size(); j++) {
255
+ Cand c;
256
+ c.beam_idx = (int)bi;
257
+ c.token = ids[j];
258
+ c.cum_logprob = b.cum_logprob + lps[j];
259
+ c.token_prob = (float)std::exp(lps[j]);
260
+ c.from_finished = false;
261
+ cands.push_back(c);
262
+ }
263
+ }
264
+
265
+ if (cands.empty())
266
+ break;
267
+
268
+ const size_t keep = std::min<size_t>((size_t)B, cands.size());
269
+ std::partial_sort(cands.begin(), cands.begin() + keep, cands.end(),
270
+ [](const Cand& a, const Cand& b) { return a.cum_logprob > b.cum_logprob; });
271
+ cands.resize(keep);
272
+
273
+ std::vector<Beam> next_beams;
274
+ next_beams.reserve(keep);
275
+ for (auto& c : cands) {
276
+ Beam nb = beams[(size_t)c.beam_idx]; // copy parent
277
+ if (c.from_finished) {
278
+ next_beams.push_back(std::move(nb));
279
+ continue;
280
+ }
281
+ nb.tokens.push_back(c.token);
282
+ nb.probs.push_back(c.token_prob);
283
+ nb.cum_logprob = c.cum_logprob;
284
+ if (is_eos(c.token))
285
+ nb.finished = true;
286
+ next_beams.push_back(std::move(nb));
287
+ }
288
+ beams = std::move(next_beams);
289
+ }
290
+
291
+ // 3. Best beam is beams[0] (top-of-cands ordering preserved by partial_sort).
292
+ result.tokens = std::move(beams[0].tokens);
293
+ result.probs = std::move(beams[0].probs);
294
+ return result;
295
+ }
296
+
297
+ // ---------------------------------------------------------------------------
298
+ // Branched variant: per-beam KV snapshots (O(B × T) single-token forwards).
299
+ // ---------------------------------------------------------------------------
300
+ //
301
+ // Use this when the backend's per-step decode is one-token-at-a-time and the
302
+ // caller can cheaply snapshot/restore the model's KV state. Gives true
303
+ // O(B × T) single-token forwards instead of replay-from-prefix's O(B × T²).
304
+ //
305
+ // Caller provides four callbacks plus the post-prefill prompt logits:
306
+ // save_fn(ctx) -> Snap (snapshot of current KV state)
307
+ // restore_fn(ctx, snap) -> void (writes snap back into ctx KV; snap
308
+ // remains valid for further restores)
309
+ // snap_free_fn(snap) -> void (called once when the snap is dead)
310
+ // step_fn(ctx, tok, n_past)-> float* (writes KV slot at n_past for `tok`,
311
+ // returns malloc'd [vocab_size] logits;
312
+ // helper free()s it. nullptr on failure.)
313
+ //
314
+ // The Snap type is whatever save_fn returns — typically a heap pointer to a
315
+ // caller-defined struct. The helper wraps it in a refcounted holder so
316
+ // siblings can share a parent's snapshot without double-free. Restore is
317
+ // read-only on the snap (it copies snap into ctx); only step_fn / save_fn
318
+ // mutate ctx KV.
319
+ //
320
+ // Pre-condition: KV slots [0, prompt_len) hold the prompt's K/V; the helper
321
+ // snapshots that state once at entry. `prefill_logits` is the [V, 1] logits
322
+ // at the last prompt position (caller-owned, not freed).
323
+ template <typename Ctx, typename SaveFn, typename RestoreFn, typename SnapFreeFn, typename StepFn>
324
+ inline Result run_with_probs_branched(Ctx* ctx, const float* prefill_logits, SaveFn save_fn, RestoreFn restore_fn,
325
+ SnapFreeFn snap_free_fn, StepFn step_fn, const Config& cfg) {
326
+ using Snap = decltype(save_fn(ctx));
327
+
328
+ Result result;
329
+ if (!prefill_logits || cfg.vocab_size <= 0)
330
+ return result;
331
+
332
+ const int B = (cfg.beam_size > 0) ? cfg.beam_size : 1;
333
+ const int V = cfg.vocab_size;
334
+
335
+ auto is_eos = [&](int id) {
336
+ if (!cfg.eos_ids.empty()) {
337
+ for (int e : cfg.eos_ids)
338
+ if (id == e)
339
+ return true;
340
+ return false;
341
+ }
342
+ return id == cfg.eos_id;
343
+ };
344
+
345
+ // RAII wrapper so siblings can share a parent's snap by shared_ptr.
346
+ // The destructor calls snap_free_fn exactly once when the last ref dies.
347
+ struct Holder {
348
+ Snap snap;
349
+ SnapFreeFn* free_cb;
350
+ Holder(Snap s, SnapFreeFn* f) : snap(s), free_cb(f) {}
351
+ ~Holder() { (*free_cb)(snap); }
352
+ Holder(const Holder&) = delete;
353
+ Holder& operator=(const Holder&) = delete;
354
+ };
355
+ auto wrap = [&snap_free_fn](Snap s) { return std::shared_ptr<Holder>(new Holder(s, &snap_free_fn)); };
356
+
357
+ struct BeamS {
358
+ std::vector<int32_t> tokens;
359
+ std::vector<float> probs;
360
+ double cum_logprob = 0.0;
361
+ bool finished = false;
362
+ // KV state right BEFORE feeding tokens.back() to step_fn. All initial
363
+ // beams share the prompt snap; siblings share their parent's
364
+ // post-step snap. Restored at the start of each per-beam expand.
365
+ std::shared_ptr<Holder> snap;
366
+ };
367
+
368
+ // 1. Snapshot the post-prefill prompt KV; seed initial beams from
369
+ // top-K of prefill_logits. No step_fn calls happen during seeding —
370
+ // the first round of the loop below will do that for each beam.
371
+ auto prompt_snap = wrap(save_fn(ctx));
372
+
373
+ std::vector<int> first_ids;
374
+ std::vector<double> first_lps;
375
+ detail::top_k_log_softmax(prefill_logits, V, B, first_ids, first_lps);
376
+
377
+ std::vector<BeamS> beams((size_t)first_ids.size());
378
+ for (size_t i = 0; i < first_ids.size(); i++) {
379
+ beams[i].tokens.push_back(first_ids[i]);
380
+ beams[i].probs.push_back((float)std::exp(first_lps[i]));
381
+ beams[i].cum_logprob = first_lps[i];
382
+ if (is_eos(first_ids[i]))
383
+ beams[i].finished = true;
384
+ beams[i].snap = prompt_snap;
385
+ }
386
+
387
+ if (beams.empty())
388
+ return result;
389
+
390
+ // 2. Per-round expand + prune.
391
+ //
392
+ // For each unfinished beam: restore beam.snap (which is the KV state
393
+ // RIGHT BEFORE feeding beam.tokens.back()), call step_fn for that token
394
+ // — which writes its KV slot and returns logits for the NEXT token —
395
+ // then capture a fresh snap (state right AFTER that token). Take top-K
396
+ // from the logits as expansion candidates; siblings inherit the same
397
+ // post-step snap shared_ptr.
398
+ while ((int)beams[0].tokens.size() < cfg.max_new_tokens && !beams[0].finished) {
399
+ struct Cand {
400
+ int beam_idx;
401
+ int token;
402
+ double cum_logprob;
403
+ float token_prob;
404
+ bool from_finished;
405
+ };
406
+ std::vector<Cand> cands;
407
+ cands.reserve((size_t)B * (size_t)B + (size_t)B);
408
+
409
+ // Per-beam post-step snap — shared by all surviving children.
410
+ std::vector<std::shared_ptr<Holder>> step_snaps(beams.size());
411
+
412
+ for (size_t bi = 0; bi < beams.size(); bi++) {
413
+ auto& b = beams[bi];
414
+ if (b.finished) {
415
+ const int sentinel = cfg.eos_ids.empty() ? cfg.eos_id : cfg.eos_ids[0];
416
+ cands.push_back({(int)bi, sentinel, b.cum_logprob, 1.0f, true});
417
+ continue;
418
+ }
419
+ // n_past = number of tokens preceding the one we're feeding.
420
+ // Beam currently holds (b.tokens.size()) chosen tokens; the
421
+ // first (b.tokens.size() - 1) of them have already been stepped
422
+ // through the LM in prior rounds. Now we step the most recent
423
+ // one at slot prompt_len + (b.tokens.size() - 1).
424
+ const int n_past = cfg.prompt_len + (int)b.tokens.size() - 1;
425
+ restore_fn(ctx, b.snap->snap);
426
+ float* lg = step_fn(ctx, b.tokens.back(), n_past);
427
+ if (!lg) {
428
+ b.finished = true;
429
+ continue;
430
+ }
431
+ step_snaps[bi] = wrap(save_fn(ctx));
432
+
433
+ std::vector<int> ids;
434
+ std::vector<double> lps;
435
+ detail::top_k_log_softmax(lg, V, B, ids, lps);
436
+ std::free(lg);
437
+
438
+ for (size_t j = 0; j < ids.size(); j++) {
439
+ cands.push_back({(int)bi, ids[j], b.cum_logprob + lps[j], (float)std::exp(lps[j]), false});
440
+ }
441
+ }
442
+
443
+ if (cands.empty())
444
+ break;
445
+
446
+ const size_t keep = std::min<size_t>((size_t)B, cands.size());
447
+ std::partial_sort(cands.begin(), cands.begin() + keep, cands.end(),
448
+ [](const Cand& a, const Cand& b) { return a.cum_logprob > b.cum_logprob; });
449
+ cands.resize(keep);
450
+
451
+ std::vector<BeamS> next_beams;
452
+ next_beams.reserve(keep);
453
+ for (auto& c : cands) {
454
+ const auto& parent = beams[(size_t)c.beam_idx];
455
+ BeamS nb;
456
+ nb.tokens = parent.tokens;
457
+ nb.probs = parent.probs;
458
+ nb.cum_logprob = parent.cum_logprob;
459
+ nb.finished = parent.finished;
460
+ if (c.from_finished) {
461
+ // Carry forward unchanged; same snap stays valid since no
462
+ // step_fn ran for this beam this round.
463
+ nb.snap = parent.snap;
464
+ next_beams.push_back(std::move(nb));
465
+ continue;
466
+ }
467
+ nb.tokens.push_back(c.token);
468
+ nb.probs.push_back(c.token_prob);
469
+ nb.cum_logprob = c.cum_logprob;
470
+ if (is_eos(c.token))
471
+ nb.finished = true;
472
+ nb.snap = step_snaps[(size_t)c.beam_idx];
473
+ next_beams.push_back(std::move(nb));
474
+ }
475
+
476
+ beams = std::move(next_beams);
477
+ // Old `beams` shared_ptrs and unselected `step_snaps` go out of scope
478
+ // here — Holder destructors fire snap_free_fn exactly once each.
479
+ }
480
+
481
+ result.tokens = std::move(beams[0].tokens);
482
+ result.probs = std::move(beams[0].probs);
483
+ return result;
484
+ }
485
+
486
+ } // namespace core_beam_decode