cohere-transcribe 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (480) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +21 -0
  3. data/LICENSE.txt +201 -0
  4. data/NOTICE +5 -0
  5. data/README.md +265 -0
  6. data/THIRD_PARTY_NOTICES.md +67 -0
  7. data/exe/cohere-transcribe +8 -0
  8. data/exe/cohere-transcribe-doctor +8 -0
  9. data/ext/cohere_transcribe_native/CMakeLists.txt +262 -0
  10. data/ext/cohere_transcribe_native/README.md +26 -0
  11. data/ext/cohere_transcribe_native/audio_abi.cpp +1416 -0
  12. data/ext/cohere_transcribe_native/audio_exports.macos +5 -0
  13. data/ext/cohere_transcribe_native/audio_exports.map +10 -0
  14. data/ext/cohere_transcribe_native/cohere_abi.cpp +1209 -0
  15. data/ext/cohere_transcribe_native/exports.macos +40 -0
  16. data/ext/cohere_transcribe_native/exports.map +45 -0
  17. data/ext/cohere_transcribe_native/extconf.rb +89 -0
  18. data/ext/cohere_transcribe_native/test/abi_smoke.rb +232 -0
  19. data/ext/cohere_transcribe_native/test/audio_matrix_smoke.cpp +121 -0
  20. data/ext/cohere_transcribe_native/test/audio_reliability_smoke.cpp +198 -0
  21. data/ext/cohere_transcribe_native/test/native_abi_reliability_smoke.cpp +186 -0
  22. data/ext/cohere_transcribe_native/test/native_batch_projection_probe.rb +81 -0
  23. data/ext/cohere_transcribe_native/test/native_cancellation_smoke.cpp +194 -0
  24. data/lib/cohere/transcribe/alignment/ATTRIBUTION.md +23 -0
  25. data/lib/cohere/transcribe/alignment/LICENSE.ctc-forced-aligner +407 -0
  26. data/lib/cohere/transcribe/alignment/LICENSE.torchaudio +25 -0
  27. data/lib/cohere/transcribe/alignment/LICENSE.uroman +26 -0
  28. data/lib/cohere/transcribe/alignment/aligner.rb +476 -0
  29. data/lib/cohere/transcribe/alignment/ctc.rb +224 -0
  30. data/lib/cohere/transcribe/alignment/text.rb +237 -0
  31. data/lib/cohere/transcribe/alignment/uroman_data.rb +4950 -0
  32. data/lib/cohere/transcribe/api.rb +173 -0
  33. data/lib/cohere/transcribe/asr/batching.rb +472 -0
  34. data/lib/cohere/transcribe/asr/failure_policy.rb +112 -0
  35. data/lib/cohere/transcribe/asr/native.rb +676 -0
  36. data/lib/cohere/transcribe/audio/ATTRIBUTION.md +8 -0
  37. data/lib/cohere/transcribe/audio/LICENSE.auditok +21 -0
  38. data/lib/cohere/transcribe/audio/decoder.rb +315 -0
  39. data/lib/cohere/transcribe/audio/ffmpeg_native.rb +248 -0
  40. data/lib/cohere/transcribe/audio/segmentation.rb +260 -0
  41. data/lib/cohere/transcribe/cli.rb +727 -0
  42. data/lib/cohere/transcribe/configuration.rb +282 -0
  43. data/lib/cohere/transcribe/constants.rb +14 -0
  44. data/lib/cohere/transcribe/dense_converter.rb +548 -0
  45. data/lib/cohere/transcribe/doctor.rb +576 -0
  46. data/lib/cohere/transcribe/errors.rb +57 -0
  47. data/lib/cohere/transcribe/gguf_writer.rb +268 -0
  48. data/lib/cohere/transcribe/hub.rb +436 -0
  49. data/lib/cohere/transcribe/input.rb +110 -0
  50. data/lib/cohere/transcribe/licenses/crispasr.txt +21 -0
  51. data/lib/cohere/transcribe/loader.rb +128 -0
  52. data/lib/cohere/transcribe/model_identity.rb +440 -0
  53. data/lib/cohere/transcribe/output/publication.rb +1118 -0
  54. data/lib/cohere/transcribe/output/rendering.rb +105 -0
  55. data/lib/cohere/transcribe/output/timing.rb +86 -0
  56. data/lib/cohere/transcribe/python_text.rb +70 -0
  57. data/lib/cohere/transcribe/pytorch_checkpoint.rb +1180 -0
  58. data/lib/cohere/transcribe/runtime/engine.rb +1676 -0
  59. data/lib/cohere/transcribe/runtime/model_provider.rb +390 -0
  60. data/lib/cohere/transcribe/runtime/precision.rb +57 -0
  61. data/lib/cohere/transcribe/runtime/preparation.rb +215 -0
  62. data/lib/cohere/transcribe/runtime/resources.rb +165 -0
  63. data/lib/cohere/transcribe/runtime/word_pipeline.rb +364 -0
  64. data/lib/cohere/transcribe/safetensors.rb +579 -0
  65. data/lib/cohere/transcribe/state/checkpoint.rb +224 -0
  66. data/lib/cohere/transcribe/state/contracts.rb +141 -0
  67. data/lib/cohere/transcribe/state/io.rb +727 -0
  68. data/lib/cohere/transcribe/state/locking.rb +211 -0
  69. data/lib/cohere/transcribe/state/manifest.rb +155 -0
  70. data/lib/cohere/transcribe/state.rb +7 -0
  71. data/lib/cohere/transcribe/types.rb +535 -0
  72. data/lib/cohere/transcribe/vad/ATTRIBUTION.md +14 -0
  73. data/lib/cohere/transcribe/vad/LICENSE.faster-whisper +21 -0
  74. data/lib/cohere/transcribe/vad/LICENSE.silero-vad +21 -0
  75. data/lib/cohere/transcribe/vad/silero.rb +344 -0
  76. data/lib/cohere/transcribe/vad/silero_vad_v6.onnx +0 -0
  77. data/lib/cohere/transcribe/vad/timestamps.rb +219 -0
  78. data/lib/cohere/transcribe/version.rb +7 -0
  79. data/lib/cohere/transcribe.rb +26 -0
  80. data/sig/cohere/transcribe.rbs +250 -0
  81. data/vendor/crispasr/AUTHORS +510 -0
  82. data/vendor/crispasr/LICENSE +21 -0
  83. data/vendor/crispasr/UPSTREAM.md +9 -0
  84. data/vendor/crispasr/VERSION +1 -0
  85. data/vendor/crispasr/ggml/AUTHORS +335 -0
  86. data/vendor/crispasr/ggml/CMakeLists.txt +512 -0
  87. data/vendor/crispasr/ggml/LICENSE +21 -0
  88. data/vendor/crispasr/ggml/README.md +49 -0
  89. data/vendor/crispasr/ggml/cmake/FindNCCL.cmake +36 -0
  90. data/vendor/crispasr/ggml/cmake/GitVars.cmake +22 -0
  91. data/vendor/crispasr/ggml/cmake/common.cmake +50 -0
  92. data/vendor/crispasr/ggml/cmake/ggml-config.cmake.in +191 -0
  93. data/vendor/crispasr/ggml/ggml.pc.in +10 -0
  94. data/vendor/crispasr/ggml/include/ggml-alloc.h +85 -0
  95. data/vendor/crispasr/ggml/include/ggml-backend.h +431 -0
  96. data/vendor/crispasr/ggml/include/ggml-blas.h +25 -0
  97. data/vendor/crispasr/ggml/include/ggml-cann.h +123 -0
  98. data/vendor/crispasr/ggml/include/ggml-cpp.h +39 -0
  99. data/vendor/crispasr/ggml/include/ggml-cpu.h +151 -0
  100. data/vendor/crispasr/ggml/include/ggml-cuda.h +50 -0
  101. data/vendor/crispasr/ggml/include/ggml-hexagon.h +19 -0
  102. data/vendor/crispasr/ggml/include/ggml-metal.h +61 -0
  103. data/vendor/crispasr/ggml/include/ggml-opencl.h +26 -0
  104. data/vendor/crispasr/ggml/include/ggml-openvino.h +37 -0
  105. data/vendor/crispasr/ggml/include/ggml-opt.h +256 -0
  106. data/vendor/crispasr/ggml/include/ggml-rpc.h +35 -0
  107. data/vendor/crispasr/ggml/include/ggml-sycl.h +49 -0
  108. data/vendor/crispasr/ggml/include/ggml-virtgpu.h +14 -0
  109. data/vendor/crispasr/ggml/include/ggml-vulkan.h +29 -0
  110. data/vendor/crispasr/ggml/include/ggml-webgpu.h +19 -0
  111. data/vendor/crispasr/ggml/include/ggml-zdnn.h +17 -0
  112. data/vendor/crispasr/ggml/include/ggml-zendnn.h +22 -0
  113. data/vendor/crispasr/ggml/include/ggml.h +2887 -0
  114. data/vendor/crispasr/ggml/include/gguf.h +204 -0
  115. data/vendor/crispasr/ggml/src/CMakeLists.txt +493 -0
  116. data/vendor/crispasr/ggml/src/ggml-alloc.c +1323 -0
  117. data/vendor/crispasr/ggml/src/ggml-backend-dl.cpp +48 -0
  118. data/vendor/crispasr/ggml/src/ggml-backend-dl.h +44 -0
  119. data/vendor/crispasr/ggml/src/ggml-backend-impl.h +275 -0
  120. data/vendor/crispasr/ggml/src/ggml-backend-meta.cpp +2145 -0
  121. data/vendor/crispasr/ggml/src/ggml-backend-reg.cpp +586 -0
  122. data/vendor/crispasr/ggml/src/ggml-backend.cpp +2437 -0
  123. data/vendor/crispasr/ggml/src/ggml-common.h +1900 -0
  124. data/vendor/crispasr/ggml/src/ggml-cpu/CMakeLists.txt +718 -0
  125. data/vendor/crispasr/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  126. data/vendor/crispasr/ggml/src/ggml-cpu/amx/amx.h +8 -0
  127. data/vendor/crispasr/ggml/src/ggml-cpu/amx/common.h +115 -0
  128. data/vendor/crispasr/ggml/src/ggml-cpu/amx/mmq.cpp +2512 -0
  129. data/vendor/crispasr/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  130. data/vendor/crispasr/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +98 -0
  131. data/vendor/crispasr/ggml/src/ggml-cpu/arch/arm/quants.c +4244 -0
  132. data/vendor/crispasr/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  133. data/vendor/crispasr/ggml/src/ggml-cpu/arch/loongarch/quants.c +2158 -0
  134. data/vendor/crispasr/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  135. data/vendor/crispasr/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  136. data/vendor/crispasr/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  137. data/vendor/crispasr/ggml/src/ggml-cpu/arch/riscv/quants.c +4455 -0
  138. data/vendor/crispasr/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  139. data/vendor/crispasr/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  140. data/vendor/crispasr/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  141. data/vendor/crispasr/ggml/src/ggml-cpu/arch/wasm/quants.c +1220 -0
  142. data/vendor/crispasr/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  143. data/vendor/crispasr/ggml/src/ggml-cpu/arch/x86/quants.c +3970 -0
  144. data/vendor/crispasr/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  145. data/vendor/crispasr/ggml/src/ggml-cpu/arch-fallback.h +349 -0
  146. data/vendor/crispasr/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  147. data/vendor/crispasr/ggml/src/ggml-cpu/binary-ops.h +16 -0
  148. data/vendor/crispasr/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  149. data/vendor/crispasr/ggml/src/ggml-cpu/common.h +95 -0
  150. data/vendor/crispasr/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  151. data/vendor/crispasr/ggml/src/ggml-cpu/ggml-cpu.c +3791 -0
  152. data/vendor/crispasr/ggml/src/ggml-cpu/ggml-cpu.cpp +703 -0
  153. data/vendor/crispasr/ggml/src/ggml-cpu/hbm.cpp +55 -0
  154. data/vendor/crispasr/ggml/src/ggml-cpu/hbm.h +8 -0
  155. data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kernels.cpp +939 -0
  156. data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kernels.h +90 -0
  157. data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1513 -0
  158. data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  159. data/vendor/crispasr/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4051 -0
  160. data/vendor/crispasr/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  161. data/vendor/crispasr/ggml/src/ggml-cpu/ops.cpp +11662 -0
  162. data/vendor/crispasr/ggml/src/ggml-cpu/ops.h +121 -0
  163. data/vendor/crispasr/ggml/src/ggml-cpu/quants.c +1288 -0
  164. data/vendor/crispasr/ggml/src/ggml-cpu/quants.h +103 -0
  165. data/vendor/crispasr/ggml/src/ggml-cpu/repack.cpp +4836 -0
  166. data/vendor/crispasr/ggml/src/ggml-cpu/repack.h +245 -0
  167. data/vendor/crispasr/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  168. data/vendor/crispasr/ggml/src/ggml-cpu/simd-mappings.h +1329 -0
  169. data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime.cpp +1025 -0
  170. data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime.h +13 -0
  171. data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +3196 -0
  172. data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime_kernels.h +26 -0
  173. data/vendor/crispasr/ggml/src/ggml-cpu/traits.cpp +36 -0
  174. data/vendor/crispasr/ggml/src/ggml-cpu/traits.h +38 -0
  175. data/vendor/crispasr/ggml/src/ggml-cpu/unary-ops.cpp +336 -0
  176. data/vendor/crispasr/ggml/src/ggml-cpu/unary-ops.h +35 -0
  177. data/vendor/crispasr/ggml/src/ggml-cpu/vec.cpp +681 -0
  178. data/vendor/crispasr/ggml/src/ggml-cpu/vec.h +1606 -0
  179. data/vendor/crispasr/ggml/src/ggml-cuda/CMakeLists.txt +272 -0
  180. data/vendor/crispasr/ggml/src/ggml-cuda/acc.cu +61 -0
  181. data/vendor/crispasr/ggml/src/ggml-cuda/acc.cuh +5 -0
  182. data/vendor/crispasr/ggml/src/ggml-cuda/add-id.cu +58 -0
  183. data/vendor/crispasr/ggml/src/ggml-cuda/add-id.cuh +3 -0
  184. data/vendor/crispasr/ggml/src/ggml-cuda/arange.cu +34 -0
  185. data/vendor/crispasr/ggml/src/ggml-cuda/arange.cuh +5 -0
  186. data/vendor/crispasr/ggml/src/ggml-cuda/argmax.cu +91 -0
  187. data/vendor/crispasr/ggml/src/ggml-cuda/argmax.cuh +3 -0
  188. data/vendor/crispasr/ggml/src/ggml-cuda/argsort.cu +265 -0
  189. data/vendor/crispasr/ggml/src/ggml-cuda/argsort.cuh +19 -0
  190. data/vendor/crispasr/ggml/src/ggml-cuda/binbcast.cu +534 -0
  191. data/vendor/crispasr/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  192. data/vendor/crispasr/ggml/src/ggml-cuda/clamp.cu +45 -0
  193. data/vendor/crispasr/ggml/src/ggml-cuda/clamp.cuh +5 -0
  194. data/vendor/crispasr/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  195. data/vendor/crispasr/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  196. data/vendor/crispasr/ggml/src/ggml-cuda/common.cuh +1489 -0
  197. data/vendor/crispasr/ggml/src/ggml-cuda/concat.cu +204 -0
  198. data/vendor/crispasr/ggml/src/ggml-cuda/concat.cuh +5 -0
  199. data/vendor/crispasr/ggml/src/ggml-cuda/conv-transpose-1d.cu +97 -0
  200. data/vendor/crispasr/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  201. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  202. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  203. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  204. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  205. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d.cu +166 -0
  206. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  207. data/vendor/crispasr/ggml/src/ggml-cuda/convert.cu +892 -0
  208. data/vendor/crispasr/ggml/src/ggml-cuda/convert.cuh +66 -0
  209. data/vendor/crispasr/ggml/src/ggml-cuda/count-equal.cu +64 -0
  210. data/vendor/crispasr/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  211. data/vendor/crispasr/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  212. data/vendor/crispasr/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  213. data/vendor/crispasr/ggml/src/ggml-cuda/cpy.cu +581 -0
  214. data/vendor/crispasr/ggml/src/ggml-cuda/cpy.cuh +7 -0
  215. data/vendor/crispasr/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  216. data/vendor/crispasr/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  217. data/vendor/crispasr/ggml/src/ggml-cuda/cumsum.cu +307 -0
  218. data/vendor/crispasr/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  219. data/vendor/crispasr/ggml/src/ggml-cuda/dequantize.cuh +99 -0
  220. data/vendor/crispasr/ggml/src/ggml-cuda/diag.cu +77 -0
  221. data/vendor/crispasr/ggml/src/ggml-cuda/diag.cuh +5 -0
  222. data/vendor/crispasr/ggml/src/ggml-cuda/diagmask.cu +40 -0
  223. data/vendor/crispasr/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  224. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-common.cuh +1212 -0
  225. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1860 -0
  226. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-tile.cu +57 -0
  227. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-tile.cuh +1309 -0
  228. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-vec.cuh +600 -0
  229. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-wmma-f16.cu +696 -0
  230. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +51 -0
  231. data/vendor/crispasr/ggml/src/ggml-cuda/fattn.cu +620 -0
  232. data/vendor/crispasr/ggml/src/ggml-cuda/fattn.cuh +5 -0
  233. data/vendor/crispasr/ggml/src/ggml-cuda/fill.cu +37 -0
  234. data/vendor/crispasr/ggml/src/ggml-cuda/fill.cuh +3 -0
  235. data/vendor/crispasr/ggml/src/ggml-cuda/gated_delta_net.cu +273 -0
  236. data/vendor/crispasr/ggml/src/ggml-cuda/gated_delta_net.cuh +4 -0
  237. data/vendor/crispasr/ggml/src/ggml-cuda/getrows.cu +332 -0
  238. data/vendor/crispasr/ggml/src/ggml-cuda/getrows.cuh +15 -0
  239. data/vendor/crispasr/ggml/src/ggml-cuda/ggml-cuda.cu +5580 -0
  240. data/vendor/crispasr/ggml/src/ggml-cuda/gla.cu +93 -0
  241. data/vendor/crispasr/ggml/src/ggml-cuda/gla.cuh +3 -0
  242. data/vendor/crispasr/ggml/src/ggml-cuda/im2col.cu +274 -0
  243. data/vendor/crispasr/ggml/src/ggml-cuda/im2col.cuh +6 -0
  244. data/vendor/crispasr/ggml/src/ggml-cuda/mean.cu +75 -0
  245. data/vendor/crispasr/ggml/src/ggml-cuda/mean.cuh +3 -0
  246. data/vendor/crispasr/ggml/src/ggml-cuda/mma.cuh +1333 -0
  247. data/vendor/crispasr/ggml/src/ggml-cuda/mmf.cu +191 -0
  248. data/vendor/crispasr/ggml/src/ggml-cuda/mmf.cuh +908 -0
  249. data/vendor/crispasr/ggml/src/ggml-cuda/mmid.cu +164 -0
  250. data/vendor/crispasr/ggml/src/ggml-cuda/mmid.cuh +5 -0
  251. data/vendor/crispasr/ggml/src/ggml-cuda/mmq.cu +372 -0
  252. data/vendor/crispasr/ggml/src/ggml-cuda/mmq.cuh +4175 -0
  253. data/vendor/crispasr/ggml/src/ggml-cuda/mmvf.cu +862 -0
  254. data/vendor/crispasr/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  255. data/vendor/crispasr/ggml/src/ggml-cuda/mmvq.cu +1161 -0
  256. data/vendor/crispasr/ggml/src/ggml-cuda/mmvq.cuh +16 -0
  257. data/vendor/crispasr/ggml/src/ggml-cuda/norm.cu +756 -0
  258. data/vendor/crispasr/ggml/src/ggml-cuda/norm.cuh +20 -0
  259. data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  260. data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  261. data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  262. data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  263. data/vendor/crispasr/ggml/src/ggml-cuda/out-prod.cu +68 -0
  264. data/vendor/crispasr/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  265. data/vendor/crispasr/ggml/src/ggml-cuda/pad.cu +106 -0
  266. data/vendor/crispasr/ggml/src/ggml-cuda/pad.cuh +5 -0
  267. data/vendor/crispasr/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  268. data/vendor/crispasr/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  269. data/vendor/crispasr/ggml/src/ggml-cuda/pool2d.cu +94 -0
  270. data/vendor/crispasr/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  271. data/vendor/crispasr/ggml/src/ggml-cuda/quantize.cu +443 -0
  272. data/vendor/crispasr/ggml/src/ggml-cuda/quantize.cuh +41 -0
  273. data/vendor/crispasr/ggml/src/ggml-cuda/reduce_rows.cuh +39 -0
  274. data/vendor/crispasr/ggml/src/ggml-cuda/roll.cu +67 -0
  275. data/vendor/crispasr/ggml/src/ggml-cuda/roll.cuh +5 -0
  276. data/vendor/crispasr/ggml/src/ggml-cuda/rope.cu +665 -0
  277. data/vendor/crispasr/ggml/src/ggml-cuda/rope.cuh +9 -0
  278. data/vendor/crispasr/ggml/src/ggml-cuda/scale.cu +34 -0
  279. data/vendor/crispasr/ggml/src/ggml-cuda/scale.cuh +5 -0
  280. data/vendor/crispasr/ggml/src/ggml-cuda/set-rows.cu +330 -0
  281. data/vendor/crispasr/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  282. data/vendor/crispasr/ggml/src/ggml-cuda/set.cu +39 -0
  283. data/vendor/crispasr/ggml/src/ggml-cuda/set.cuh +7 -0
  284. data/vendor/crispasr/ggml/src/ggml-cuda/softcap.cu +34 -0
  285. data/vendor/crispasr/ggml/src/ggml-cuda/softcap.cuh +5 -0
  286. data/vendor/crispasr/ggml/src/ggml-cuda/softmax.cu +472 -0
  287. data/vendor/crispasr/ggml/src/ggml-cuda/softmax.cuh +7 -0
  288. data/vendor/crispasr/ggml/src/ggml-cuda/solve_tri.cu +275 -0
  289. data/vendor/crispasr/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  290. data/vendor/crispasr/ggml/src/ggml-cuda/ssm-conv.cu +197 -0
  291. data/vendor/crispasr/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  292. data/vendor/crispasr/ggml/src/ggml-cuda/ssm-scan.cu +342 -0
  293. data/vendor/crispasr/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  294. data/vendor/crispasr/ggml/src/ggml-cuda/sum.cu +41 -0
  295. data/vendor/crispasr/ggml/src/ggml-cuda/sum.cuh +5 -0
  296. data/vendor/crispasr/ggml/src/ggml-cuda/sumrows.cu +43 -0
  297. data/vendor/crispasr/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  298. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
  299. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  300. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +11 -0
  301. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  302. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
  303. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  304. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
  305. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  306. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  307. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +11 -0
  308. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  309. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
  310. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
  311. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
  312. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  313. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +11 -0
  314. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  315. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  316. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
  317. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  318. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +11 -0
  319. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  320. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  321. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  322. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  323. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  324. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  325. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  326. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  327. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  328. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  329. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  330. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  331. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  332. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  333. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  334. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  335. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  336. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  337. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  338. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  339. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  340. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  341. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  342. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  343. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  344. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  345. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  346. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  347. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  348. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  349. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  350. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  351. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  352. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  353. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  354. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  355. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  356. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  357. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  358. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  359. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  360. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  361. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  362. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  363. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  364. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  365. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  366. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  367. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  368. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  369. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  370. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  371. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  372. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  373. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  374. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  375. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  376. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  377. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  378. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  379. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  380. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  381. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  382. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  383. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  384. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  385. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  386. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  387. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  388. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  389. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  390. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  391. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  392. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  393. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  394. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  395. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  396. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  397. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  398. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  399. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  400. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  401. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  402. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  403. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  404. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  405. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  406. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  407. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  408. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  409. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  410. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  411. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  412. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  413. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  414. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  415. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  416. data/vendor/crispasr/ggml/src/ggml-cuda/top-k.cu +95 -0
  417. data/vendor/crispasr/ggml/src/ggml-cuda/top-k.cuh +3 -0
  418. data/vendor/crispasr/ggml/src/ggml-cuda/topk-moe.cu +415 -0
  419. data/vendor/crispasr/ggml/src/ggml-cuda/topk-moe.cuh +27 -0
  420. data/vendor/crispasr/ggml/src/ggml-cuda/tri.cu +136 -0
  421. data/vendor/crispasr/ggml/src/ggml-cuda/tri.cuh +5 -0
  422. data/vendor/crispasr/ggml/src/ggml-cuda/tsembd.cu +47 -0
  423. data/vendor/crispasr/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  424. data/vendor/crispasr/ggml/src/ggml-cuda/unary.cu +644 -0
  425. data/vendor/crispasr/ggml/src/ggml-cuda/unary.cuh +116 -0
  426. data/vendor/crispasr/ggml/src/ggml-cuda/upscale.cu +293 -0
  427. data/vendor/crispasr/ggml/src/ggml-cuda/upscale.cuh +5 -0
  428. data/vendor/crispasr/ggml/src/ggml-cuda/vecdotq.cuh +1317 -0
  429. data/vendor/crispasr/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  430. data/vendor/crispasr/ggml/src/ggml-cuda/vendors/hip.h +306 -0
  431. data/vendor/crispasr/ggml/src/ggml-cuda/vendors/musa.h +148 -0
  432. data/vendor/crispasr/ggml/src/ggml-cuda/wkv.cu +199 -0
  433. data/vendor/crispasr/ggml/src/ggml-cuda/wkv.cuh +7 -0
  434. data/vendor/crispasr/ggml/src/ggml-impl.h +783 -0
  435. data/vendor/crispasr/ggml/src/ggml-metal/CMakeLists.txt +124 -0
  436. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-common.cpp +457 -0
  437. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-common.h +52 -0
  438. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  439. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-context.m +778 -0
  440. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-device.cpp +2117 -0
  441. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-device.h +299 -0
  442. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-device.m +2093 -0
  443. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-impl.h +1267 -0
  444. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-ops.cpp +5023 -0
  445. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-ops.h +111 -0
  446. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal.cpp +954 -0
  447. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal.metal +11756 -0
  448. data/vendor/crispasr/ggml/src/ggml-opt.cpp +1094 -0
  449. data/vendor/crispasr/ggml/src/ggml-quants.c +5491 -0
  450. data/vendor/crispasr/ggml/src/ggml-quants.h +112 -0
  451. data/vendor/crispasr/ggml/src/ggml-threading.cpp +12 -0
  452. data/vendor/crispasr/ggml/src/ggml-threading.h +14 -0
  453. data/vendor/crispasr/ggml/src/ggml.c +7925 -0
  454. data/vendor/crispasr/ggml/src/ggml.cpp +26 -0
  455. data/vendor/crispasr/ggml/src/gguf.cpp +1556 -0
  456. data/vendor/crispasr/src/cohere-arch.h +137 -0
  457. data/vendor/crispasr/src/cohere.cpp +5642 -0
  458. data/vendor/crispasr/src/cohere.h +327 -0
  459. data/vendor/crispasr/src/cohere_batch_planner.h +82 -0
  460. data/vendor/crispasr/src/cohere_chunking.h +64 -0
  461. data/vendor/crispasr/src/cohere_decoder_batch_layout.h +60 -0
  462. data/vendor/crispasr/src/cohere_encoder_padded_layout.h +27 -0
  463. data/vendor/crispasr/src/cohere_frontend.cpp +189 -0
  464. data/vendor/crispasr/src/cohere_frontend.h +31 -0
  465. data/vendor/crispasr/src/cohere_ragged_controller.h +149 -0
  466. data/vendor/crispasr/src/cohere_token_renderer.h +181 -0
  467. data/vendor/crispasr/src/core/attention.h +924 -0
  468. data/vendor/crispasr/src/core/audio_chunking.h +97 -0
  469. data/vendor/crispasr/src/core/beam_decode.h +486 -0
  470. data/vendor/crispasr/src/core/cpu_ops.h +135 -0
  471. data/vendor/crispasr/src/core/gguf_loader.cpp +1021 -0
  472. data/vendor/crispasr/src/core/gguf_loader.h +216 -0
  473. data/vendor/crispasr/src/core/gpu_backend_pref.h +119 -0
  474. data/vendor/crispasr/src/core/mel.cpp +519 -0
  475. data/vendor/crispasr/src/core/mel.h +265 -0
  476. data/vendor/crispasr/src/core/ngram_loop_fix.h +173 -0
  477. data/vendor/crispasr/src/core/repetition_loop_guard.h +54 -0
  478. data/vendor/crispasr/src/crispasr_imatrix.cpp +255 -0
  479. data/vendor/crispasr/src/crispasr_imatrix.h +38 -0
  480. metadata +596 -0
@@ -0,0 +1,576 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "optparse"
4
+ require_relative "version"
5
+
6
+ module Cohere
7
+ module Transcribe
8
+ # Installation and model-metadata diagnostics that never load ASR weights.
9
+ module Doctor
10
+ DEFAULT_MODEL_ID = "CohereLabs/cohere-transcribe-arabic-07-2026"
11
+ EXPECTED_ONNX_SHA256 = "914fd98ac0a73d69ba1e70c9b1d66acb740eff90500dfde08b89a961b168a6a9"
12
+ COHERE_PROCESSOR_CLASS = "CohereAsrProcessor"
13
+ COHERE_FEATURE_EXTRACTOR_CLASS = "CohereAsrFeatureExtractor"
14
+ DEFAULT_MAX_AUDIO_CLIP_SECONDS = 35.0
15
+ REQUIRED_PROMPT_TOKENS = %w[
16
+ ▁ <|startofcontext|> <|startoftranscript|> <|emo:undefined|>
17
+ <|ar|> <|en|> <|pnc|> <|noitn|> <|notimestamp|> <|nodiarize|>
18
+ <|endoftext|>
19
+ ].freeze
20
+ VALUE_OPTIONS = %w[mode model model-revision adapter adapter-revision audio-backend].freeze
21
+ FLAG_OPTIONS = %w[model-access help].freeze
22
+ private_constant :VALUE_OPTIONS, :FLAG_OPTIONS
23
+
24
+ Options = Data.define(
25
+ :mode,
26
+ :model_access,
27
+ :model,
28
+ :model_revision,
29
+ :adapter,
30
+ :adapter_revision,
31
+ :audio_backend
32
+ )
33
+
34
+ class EarlyExit < StandardError
35
+ attr_reader :status
36
+
37
+ def initialize(status = 0)
38
+ @status = status
39
+ super()
40
+ end
41
+ end
42
+
43
+ class Results
44
+ attr_reader :failures, :warnings
45
+
46
+ def initialize(out: $stdout)
47
+ @out = out
48
+ @failures = 0
49
+ @warnings = 0
50
+ end
51
+
52
+ def ok(message)
53
+ @out.puts("[OK] #{message}")
54
+ end
55
+
56
+ def warn(message)
57
+ @warnings += 1
58
+ @out.puts("[WARN] #{message}")
59
+ end
60
+
61
+ def fail(message)
62
+ @failures += 1
63
+ @out.puts("[FAIL] #{message}")
64
+ end
65
+ end
66
+
67
+ module_function
68
+
69
+ def parse_args(argv = ARGV, out: $stdout)
70
+ values = {
71
+ mode: "segment",
72
+ model_access: false,
73
+ model: DEFAULT_MODEL_ID,
74
+ model_revision: nil,
75
+ adapter: nil,
76
+ adapter_revision: nil,
77
+ audio_backend: "auto"
78
+ }
79
+ raw_arguments = Array(argv).dup
80
+ preflight_ambiguous_long_options!(raw_arguments)
81
+ arguments = defer_unknown_options_before_early_exit(raw_arguments)
82
+ parser = option_parser(values, out: out)
83
+ parser.parse!(arguments)
84
+ raise OptionParser::InvalidArgument, "unexpected positional argument: #{arguments.first}" unless arguments.empty?
85
+
86
+ Options.new(**values)
87
+ end
88
+
89
+ def main(argv = ARGV, out: $stdout, err: $stderr, checks: nil)
90
+ options = parse_args(argv, out: out)
91
+ checker = checks || self
92
+ results = Results.new(out: out)
93
+ checker.validate_files(results)
94
+ checker.validate_common_runtime(results)
95
+ checker.validate_silero(results)
96
+ checker.validate_word_alignment(results) if options.mode == "word"
97
+ checker.report_optional_runtime(results, options.audio_backend)
98
+
99
+ selected_model = options.model != DEFAULT_MODEL_ID ||
100
+ !options.model_revision.nil? ||
101
+ !options.adapter.nil? ||
102
+ !options.adapter_revision.nil?
103
+ if options.model_access || selected_model
104
+ checker.validate_model_access(
105
+ results,
106
+ include_aligner: options.mode == "word",
107
+ model_id: options.model,
108
+ model_revision: options.model_revision,
109
+ adapter_id: options.adapter,
110
+ adapter_revision: options.adapter_revision
111
+ )
112
+ end
113
+
114
+ out.puts
115
+ if results.failures.positive?
116
+ out.puts("Validation failed: #{results.failures} failure(s), #{results.warnings} warning(s).")
117
+ 1
118
+ else
119
+ out.puts("Validation passed for #{options.mode} mode with #{results.warnings} warning(s).")
120
+ 0
121
+ end
122
+ rescue EarlyExit => e
123
+ e.status
124
+ rescue OptionParser::ParseError => e
125
+ err.puts(option_parser({}, out: out))
126
+ err.puts("cohere-transcribe-doctor: error: #{e.message}")
127
+ 2
128
+ end
129
+
130
+ def option_parser(values, out:)
131
+ OptionParser.new do |parser|
132
+ # OptionParser installs an implicit --version switch that aborts when
133
+ # no program version is configured. The reference doctor does not
134
+ # expose that switch, so remove it and let normal unknown-option
135
+ # handling report the command-line error.
136
+ parser.base.long.delete("version")
137
+ parser.banner = "Usage: cohere-transcribe-doctor [options]"
138
+ parser.separator("")
139
+ parser.separator("Validate the transcription package without loading model weights.")
140
+ parser.separator("")
141
+ parser.on("--mode MODE", String, %w[word segment], "Output mode to validate (default: segment).") do |value|
142
+ values[:mode] = value
143
+ end
144
+ parser.on("--model-access", "Resolve and validate selected model metadata.") do
145
+ values[:model_access] = true
146
+ end
147
+ parser.on("--model MODEL", "Hub repository or local model directory; implies model access.") do |value|
148
+ values[:model] = value
149
+ end
150
+ parser.on("--model-revision REVISION", "Optional Hub model commit, tag, or branch.") do |value|
151
+ values[:model_revision] = value
152
+ end
153
+ parser.on("--adapter ADAPTER", "Optional Hub repository or local LoRA adapter directory.") do |value|
154
+ values[:adapter] = value
155
+ end
156
+ parser.on("--adapter-revision REVISION", "Optional Hub adapter commit, tag, or branch.") do |value|
157
+ values[:adapter_revision] = value
158
+ end
159
+ parser.on(
160
+ "--audio-backend BACKEND",
161
+ String,
162
+ %w[auto torchcodec ffmpeg librosa],
163
+ "Decoder configuration to validate (default: auto)."
164
+ ) { |value| values[:audio_backend] = value }
165
+ parser.on_tail("-h", "--help", "Show this help and exit.") do
166
+ out.puts(parser)
167
+ raise EarlyExit, 0
168
+ end
169
+ end
170
+ end
171
+
172
+ def preflight_ambiguous_long_options!(arguments)
173
+ arguments.each do |argument|
174
+ break if argument == "--"
175
+ next unless argument.start_with?("--") && argument.length > 2
176
+
177
+ supplied = argument.delete_prefix("--").split("=", 2).first
178
+ names = VALUE_OPTIONS + FLAG_OPTIONS
179
+ candidates = names.select { |name| name.start_with?(supplied) }
180
+ canonical = if names.include?(supplied)
181
+ supplied
182
+ elsif candidates.one?
183
+ candidates.first
184
+ end
185
+ break if canonical == "help"
186
+ next if names.include?(supplied)
187
+
188
+ raise OptionParser::AmbiguousOption, "--#{supplied}" if candidates.length > 1
189
+ end
190
+ end
191
+ private_class_method :preflight_ambiguous_long_options!
192
+
193
+ def defer_unknown_options_before_early_exit(arguments)
194
+ unknown_indices = []
195
+ early_index = nil
196
+ index = 0
197
+ while index < arguments.length
198
+ argument = arguments[index]
199
+ break if argument == "--"
200
+
201
+ option = canonical_long_option(argument)
202
+ if argument == "-h" || option == "help"
203
+ early_index = index
204
+ break
205
+ end
206
+ if argument.start_with?("--") && long_option_candidates(argument).empty?
207
+ unknown_indices << index
208
+ elsif argument.start_with?("-") && argument != "-" && !argument.start_with?("--")
209
+ unknown_indices << index
210
+ end
211
+
212
+ index += 1 if VALUE_OPTIONS.include?(option) && !argument.include?("=")
213
+ index += 1
214
+ end
215
+ return arguments.dup if early_index.nil? || unknown_indices.empty?
216
+
217
+ deferred = unknown_indices.map { |unknown_index| arguments[unknown_index] }
218
+ arguments.each_with_index.filter_map do |argument, argument_index|
219
+ next if unknown_indices.include?(argument_index)
220
+
221
+ argument_index == early_index ? [argument, *deferred] : argument
222
+ end.flatten
223
+ end
224
+ private_class_method :defer_unknown_options_before_early_exit
225
+
226
+ def canonical_long_option(argument)
227
+ return nil unless argument.start_with?("--") && argument.length > 2
228
+
229
+ supplied = argument.delete_prefix("--").split("=", 2).first
230
+ names = VALUE_OPTIONS + FLAG_OPTIONS
231
+ return supplied if names.include?(supplied)
232
+
233
+ candidates = long_option_candidates(argument)
234
+ candidates.one? ? candidates.first : nil
235
+ end
236
+ private_class_method :canonical_long_option
237
+
238
+ def long_option_candidates(argument)
239
+ return [] unless argument.start_with?("--") && argument.length > 2
240
+
241
+ supplied = argument.delete_prefix("--").split("=", 2).first
242
+ (VALUE_OPTIONS + FLAG_OPTIONS).select { |name| name.start_with?(supplied) }
243
+ end
244
+ private_class_method :long_option_candidates
245
+
246
+ def validate_files(results)
247
+ require "digest"
248
+
249
+ specification = Gem.loaded_specs["cohere-transcribe"]
250
+ if specification.nil?
251
+ results.warn("package metadata is unavailable; running from a source checkout")
252
+ elsif specification.version.to_s == VERSION
253
+ results.ok("package metadata version: #{VERSION}")
254
+ else
255
+ results.fail("package metadata is #{specification.version}, runtime is #{VERSION}")
256
+ end
257
+
258
+ asset = File.expand_path("vad/silero_vad_v6.onnx", __dir__)
259
+ unless File.file?(asset)
260
+ results.fail("missing Silero ONNX asset: #{asset}")
261
+ return
262
+ end
263
+ digest = Digest::SHA256.file(asset).hexdigest
264
+ if digest == EXPECTED_ONNX_SHA256
265
+ results.ok("Silero ONNX asset integrity: #{digest}")
266
+ else
267
+ results.fail(
268
+ "Silero ONNX checksum mismatch: expected #{EXPECTED_ONNX_SHA256}, found #{digest}"
269
+ )
270
+ end
271
+ rescue SystemCallError => e
272
+ results.fail("cannot validate packaged assets: #{e.class}: #{e.message}")
273
+ end
274
+
275
+ def validate_common_runtime(results, native_library: nil)
276
+ results.ok("Ruby #{RUBY_VERSION}")
277
+ import_required(results, "numo/narray", "numeric runtime")
278
+ import_required(results, "onnxruntime", "ONNX inference runtime")
279
+ require_relative "constants"
280
+ require_relative "errors"
281
+ require_relative "asr/native"
282
+ library = native_library || ASR::NativeLibrary.load
283
+ results.ok("native Cohere ASR runtime: #{library.path}")
284
+ report_native_device_capabilities(results, library)
285
+ rescue StandardError => e
286
+ results.fail("native Cohere ASR runtime: #{e.class}: #{e.message}")
287
+ end
288
+
289
+ def report_native_device_capabilities(results, library)
290
+ available = ["cpu"]
291
+ %w[cuda mps].each do |device|
292
+ available << device if library.resolve_device(device) == device
293
+ rescue TranscriptionRuntimeError
294
+ next
295
+ end
296
+ resolved = library.resolve_device("auto")
297
+ results.ok("native inference devices: #{available.join(", ")}; auto resolves to #{resolved}")
298
+ case resolved
299
+ when "cuda"
300
+ bf16 = library.supports_bf16?(resolved) ? "supported" : "not supported"
301
+ results.ok("accelerator: CUDA available through native runtime; BF16 operations #{bf16}")
302
+ when "mps"
303
+ bf16 = library.supports_bf16?(resolved) ? "supported" : "not supported"
304
+ results.ok("accelerator: Apple Metal available through native runtime; BF16 operations #{bf16}")
305
+ when "cpu"
306
+ results.warn("accelerator: CPU only; the 2B model will be substantially slower")
307
+ else
308
+ results.fail("native runtime returned an unsupported automatic device: #{resolved.inspect}")
309
+ end
310
+ end
311
+
312
+ def validate_silero(results)
313
+ require_relative "vad/silero"
314
+
315
+ probabilities = VAD::Silero.new.speech_probabilities(Array.new(1024, 0.0))
316
+ valid = probabilities.length == 2 && probabilities.all? do |probability|
317
+ probability.is_a?(Numeric) && probability.finite? && probability.between?(0.0, 1.0)
318
+ end
319
+ if valid
320
+ results.ok("Silero ONNX smoke: 2 finite probability frames")
321
+ else
322
+ results.fail("Silero ONNX smoke returned invalid probabilities: #{probabilities.inspect}")
323
+ end
324
+ rescue LoadError, StandardError => e
325
+ results.fail("Silero ONNX smoke: #{e.class}: #{e.message}")
326
+ end
327
+
328
+ def validate_word_alignment(results)
329
+ require_relative "alignment/aligner"
330
+
331
+ path = Alignment::CTC.forced_align(
332
+ Numo::SFloat.cast([[4.0, 0.0], [0.0, 4.0]]),
333
+ [1],
334
+ blank: 0
335
+ )
336
+ raise "unexpected CTC path #{path.inspect}" unless path == [0, 1]
337
+
338
+ tokens, = Alignment::Text.preprocess("مرحبا بكم في العالم", "ara")
339
+ expected = ["<star>", "m r h b a", "<star>", "b k m", "<star>", "f y", "<star>", "a l ' a l m"]
340
+ raise "unexpected Arabic romanization #{tokens.inspect}" unless tokens == expected
341
+
342
+ fp32 = Alignment::ModelProvider::ARTIFACTS.fetch("fp32")
343
+ results.ok(
344
+ "word alignment: pure-Ruby MMS CTC Viterbi and Arabic romanization smokes execute"
345
+ )
346
+ results.ok(
347
+ "word aligner: #{Alignment::ModelProvider::REPOSITORY}@#{Alignment::ModelProvider::REVISION}, " \
348
+ "FP32 SHA-256 #{fp32.sha256}, bounded per-segment uniform fallback"
349
+ )
350
+ rescue StandardError => e
351
+ results.fail("word alignment: #{e.class}: #{e.message}")
352
+ end
353
+
354
+ def report_optional_runtime(results, audio_backend)
355
+ case audio_backend
356
+ when "auto", "ffmpeg", "torchcodec", "librosa"
357
+ require_relative "audio/decoder"
358
+ if Audio::FFmpegNative.available?
359
+ results.ok("native FFmpeg decoder: #{Audio::FFmpegNative.diagnostic}")
360
+ if %w[torchcodec librosa].include?(audio_backend)
361
+ results.warn("#{audio_backend} compatibility mode uses FFmpeg through the native C ABI")
362
+ end
363
+ else
364
+ sound_file = Audio.const_get(:SoundFileABI, false)
365
+ if %w[auto librosa].include?(audio_backend) && sound_file.const_get(:AVAILABLE)
366
+ label = audio_backend == "librosa" ? "librosa compatibility fallback" : "native audio decoder fallback"
367
+ results.ok("#{label}: libsndfile ABI")
368
+ results.warn("native FFmpeg decoder unavailable: #{Audio::FFmpegNative.diagnostic}")
369
+ else
370
+ results.fail("native FFmpeg decoder unavailable: #{Audio::FFmpegNative.diagnostic}")
371
+ end
372
+ end
373
+ end
374
+ rescue LoadError, StandardError => e
375
+ results.fail("audio decoder configuration: #{e.class}: #{e.message}")
376
+ end
377
+
378
+ def validate_model_access(
379
+ results,
380
+ include_aligner:,
381
+ model_id: DEFAULT_MODEL_ID,
382
+ model_revision: nil,
383
+ adapter_id: nil,
384
+ adapter_revision: nil,
385
+ hub: nil
386
+ )
387
+ require_relative "constants"
388
+ require_relative "hub"
389
+ require_relative "model_identity"
390
+
391
+ hub ||= Hub.new
392
+ identity = ModelIdentity.resolve(
393
+ model_id,
394
+ model_revision,
395
+ adapter_id,
396
+ adapter_revision,
397
+ hub: hub
398
+ )
399
+ maximum = validate_asr_processor_metadata!(identity, hub)
400
+ validate_aligner_metadata!(hub) if include_aligner
401
+ model_reference = reference(identity.model_id, identity.model_revision)
402
+ adapter = if identity.adapter_id
403
+ ", adapter #{reference(identity.adapter_id, identity.adapter_revision)}"
404
+ else
405
+ ""
406
+ end
407
+ aligner = include_aligner ? " and pinned MMS ONNX aligner metadata" : ""
408
+ results.ok(
409
+ "ASR configuration #{model_reference} (#{identity.model_format})#{adapter}#{aligner} accessible; " \
410
+ "#{COHERE_PROCESSOR_CLASS} one-row limit is #{maximum}s"
411
+ )
412
+ if identity.model_format.to_s != "dense"
413
+ results.warn(
414
+ "saved #{identity.model_format} checkpoint detected; quantized inference is not part of the core Ruby path"
415
+ )
416
+ end
417
+ results.warn("PEFT/LoRA adapter inference is not part of the core Ruby Dense path") if identity.adapter_id
418
+ rescue Hub::AuthenticationError => e
419
+ results.fail(
420
+ "Cannot access the gated ASR model #{model_id}. Accept its terms, then set HF_TOKEN. " \
421
+ "(#{e.class}: #{e.message})"
422
+ )
423
+ rescue StandardError => e
424
+ results.fail("ASR model or adapter validation: #{e.class}: #{e.message}")
425
+ end
426
+
427
+ def import_required(results, library, feature)
428
+ require library
429
+ gem_name = { "numo/narray" => "numo-narray" }.fetch(library, library.split("/").first)
430
+ version = Gem.loaded_specs[gem_name]&.version
431
+ suffix = version ? " #{version}" : ""
432
+ results.ok("#{feature}: #{library}#{suffix}")
433
+ true
434
+ rescue LoadError => e
435
+ results.fail("#{feature}: cannot require #{library.inspect}: #{e.message}")
436
+ false
437
+ end
438
+
439
+ def find_executable(name)
440
+ ENV.fetch("PATH", "").split(File::PATH_SEPARATOR).filter_map do |directory|
441
+ path = File.join(directory, name)
442
+ path if File.file?(path) && File.executable?(path)
443
+ end.first
444
+ end
445
+ private_class_method :find_executable
446
+
447
+ def reference(identifier, revision)
448
+ revision ? "#{identifier}@#{revision}" : identifier
449
+ end
450
+ private_class_method :reference
451
+
452
+ def validate_asr_processor_metadata!(identity, hub)
453
+ config = read_model_artifact(identity, hub, "config.json")
454
+ preprocessor = read_model_artifact(identity, hub, "preprocessor_config.json")
455
+ tokenizer = read_model_artifact(identity, hub, "tokenizer.json")
456
+
457
+ unless config["model_type"] == "cohere_asr"
458
+ raise ArgumentError,
459
+ "config.json declares model_type=#{config["model_type"].inspect}; expected cohere_asr"
460
+ end
461
+
462
+ processor_class = preprocessor["processor_class"]
463
+ unless processor_class == COHERE_PROCESSOR_CLASS
464
+ raise ArgumentError,
465
+ "preprocessor_config.json declares processor_class=#{processor_class.inspect}; " \
466
+ "expected #{COHERE_PROCESSOR_CLASS}"
467
+ end
468
+ extractor_class = preprocessor["feature_extractor_type"]
469
+ if extractor_class && extractor_class != COHERE_FEATURE_EXTRACTOR_CLASS
470
+ raise ArgumentError,
471
+ "preprocessor_config.json declares feature_extractor_type=#{extractor_class.inspect}; " \
472
+ "expected #{COHERE_FEATURE_EXTRACTOR_CLASS}"
473
+ end
474
+
475
+ maximum = preprocessor.fetch("max_audio_clip_s", DEFAULT_MAX_AUDIO_CLIP_SECONDS)
476
+ unless maximum.is_a?(Numeric) && !maximum.is_a?(Complex) && maximum.finite? && maximum.positive?
477
+ raise ArgumentError,
478
+ "Cohere processor reported an invalid max_audio_clip_s: #{maximum.inspect}"
479
+ end
480
+ validate_tokenizer_metadata!(tokenizer, config)
481
+ Float(maximum)
482
+ end
483
+ private_class_method :validate_asr_processor_metadata!
484
+
485
+ def read_model_artifact(identity, hub, filename)
486
+ path = if identity.model_revision.nil?
487
+ Pathname(identity.model_id).join(filename)
488
+ else
489
+ hub.download(identity.model_id, filename, revision: identity.model_revision)
490
+ end
491
+ ModelIdentity.read_json_object(path)
492
+ end
493
+ private_class_method :read_model_artifact
494
+
495
+ def validate_tokenizer_metadata!(tokenizer, config)
496
+ vocabulary = tokenizer.dig("model", "vocab")
497
+ raise ArgumentError, "tokenizer.json has no model vocabulary" unless vocabulary.is_a?(Hash) && !vocabulary.empty?
498
+
499
+ added_tokens = tokenizer.fetch("added_tokens", [])
500
+ raise ArgumentError, "tokenizer.json added_tokens must be an array" unless added_tokens.is_a?(Array)
501
+
502
+ tokens_by_id = {}
503
+ ids_by_token = {}
504
+ vocabulary.each do |token, id|
505
+ add_tokenizer_entry!(tokens_by_id, ids_by_token, token, id)
506
+ end
507
+ added_tokens.each do |entry|
508
+ raise ArgumentError, "tokenizer.json contains an invalid added token" unless entry.is_a?(Hash)
509
+
510
+ add_tokenizer_entry!(tokens_by_id, ids_by_token, entry["content"], entry["id"])
511
+ end
512
+
513
+ expected_size = config["vocab_size"] || config.dig("head", "num_classes")
514
+ if expected_size
515
+ unless expected_size.is_a?(Integer) && expected_size.positive?
516
+ raise ArgumentError, "model configuration has an invalid vocabulary size: #{expected_size.inspect}"
517
+ end
518
+
519
+ missing_ids = (0...expected_size).reject { |id| tokens_by_id.key?(id) }
520
+ unless missing_ids.empty?
521
+ raise ArgumentError,
522
+ "tokenizer.json vocabulary has missing token IDs: #{missing_ids.first(8).inspect}"
523
+ end
524
+ outside = tokens_by_id.keys.select { |id| id >= expected_size }
525
+ unless outside.empty?
526
+ raise ArgumentError,
527
+ "tokenizer.json token ID #{outside.min} is outside the checkpoint vocabulary size #{expected_size}"
528
+ end
529
+ end
530
+
531
+ missing_prompt = REQUIRED_PROMPT_TOKENS.reject { |token| ids_by_token.key?(token) }
532
+ return if missing_prompt.empty?
533
+
534
+ raise ArgumentError,
535
+ "tokenizer.json is missing Cohere prompt tokens: #{missing_prompt.join(", ")}"
536
+ end
537
+ private_class_method :validate_tokenizer_metadata!
538
+
539
+ def add_tokenizer_entry!(tokens_by_id, ids_by_token, token, id)
540
+ raise ArgumentError, "tokenizer.json contains an invalid token/id entry" unless token.is_a?(String) && id.is_a?(Integer) && id >= 0
541
+
542
+ previous_token = tokens_by_id[id]
543
+ if previous_token && previous_token != token
544
+ raise ArgumentError,
545
+ "tokenizer.json assigns ID #{id} to both #{previous_token.inspect} and #{token.inspect}"
546
+ end
547
+ previous_id = ids_by_token[token]
548
+ if previous_id && previous_id != id
549
+ raise ArgumentError,
550
+ "tokenizer.json assigns token #{token.inspect} to IDs #{previous_id} and #{id}"
551
+ end
552
+
553
+ tokens_by_id[id] = token
554
+ ids_by_token[token] = id
555
+ end
556
+ private_class_method :add_tokenizer_entry!
557
+
558
+ def validate_aligner_metadata!(hub)
559
+ require_relative "alignment/aligner"
560
+
561
+ provider = Alignment::ModelProvider
562
+ config_path = hub.download(provider::REPOSITORY, "config.json", revision: provider::REVISION)
563
+ vocabulary_path = hub.download(provider::REPOSITORY, "vocab.json", revision: provider::REVISION)
564
+ config = ModelIdentity.read_json_object(config_path)
565
+ vocabulary = ModelIdentity.read_json_object(vocabulary_path)
566
+ stride = Array(config["conv_stride"]).reduce(1) { |product, value| product * Integer(value) }
567
+
568
+ raise "pinned aligner input stride changed" unless stride == Alignment::Aligner::INPUTS_TO_LOGITS_RATIO
569
+ raise "pinned aligner vocabulary size changed" unless config["vocab_size"] == Alignment::Aligner::VOCABULARY.length
570
+ raise "pinned aligner source checkpoint changed" unless config["_name_or_path"] == provider::SOURCE_REPOSITORY
571
+ raise "pinned aligner vocabulary changed" unless vocabulary == Alignment::Aligner::VOCABULARY
572
+ end
573
+ private_class_method :validate_aligner_metadata!
574
+ end
575
+ end
576
+ end
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cohere
4
+ module Transcribe
5
+ # Keep the generated gem's original base error compatible when this file is
6
+ # loaded directly as well as through the eventual public entry point.
7
+ class Error < StandardError; end unless const_defined?(:Error, false)
8
+
9
+ # Base exception for programmatic transcription failures.
10
+ class TranscriptionError < Error; end
11
+
12
+ # Invalid or unsupported transcription configuration.
13
+ class TranscriptionConfigurationError < TranscriptionError; end
14
+
15
+ # Invalid audio input or output planning request.
16
+ class TranscriptionInputError < TranscriptionError; end
17
+
18
+ # Dependency, device, model, or execution initialization failure.
19
+ class TranscriptionRuntimeError < TranscriptionError; end
20
+
21
+ # A user-provided progress callback raised an exception.
22
+ class ProgressCallbackError < TranscriptionError
23
+ attr_reader :original
24
+
25
+ def initialize(original)
26
+ @original = original
27
+ super("Progress callback failed: #{original.class}: #{original}")
28
+ end
29
+ end
30
+
31
+ # Operation attempted after a transcriber was closed.
32
+ class TranscriberClosedError < TranscriptionError; end
33
+
34
+ # A reentrant or otherwise conflicting transcriber operation was attempted.
35
+ class TranscriberBusyError < TranscriptionError; end
36
+
37
+ # Aggregate failure that retains every completed per-file result.
38
+ class BatchTranscriptionError < TranscriptionError
39
+ attr_reader :run
40
+
41
+ def initialize(run)
42
+ @run = run
43
+ failed_count = run.failed.length
44
+ run_error_count = run.errors.length
45
+ message = if failed_count.positive? && run_error_count.positive?
46
+ "#{failed_count} transcription file(s) failed; " \
47
+ "run errors: #{run_error_count}"
48
+ elsif failed_count.positive?
49
+ "#{failed_count} transcription file(s) failed"
50
+ else
51
+ "Transcription run failed with #{run_error_count} run error(s)"
52
+ end
53
+ super(message)
54
+ end
55
+ end
56
+ end
57
+ end