cohere-transcribe 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (480) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +21 -0
  3. data/LICENSE.txt +201 -0
  4. data/NOTICE +5 -0
  5. data/README.md +265 -0
  6. data/THIRD_PARTY_NOTICES.md +67 -0
  7. data/exe/cohere-transcribe +8 -0
  8. data/exe/cohere-transcribe-doctor +8 -0
  9. data/ext/cohere_transcribe_native/CMakeLists.txt +262 -0
  10. data/ext/cohere_transcribe_native/README.md +26 -0
  11. data/ext/cohere_transcribe_native/audio_abi.cpp +1416 -0
  12. data/ext/cohere_transcribe_native/audio_exports.macos +5 -0
  13. data/ext/cohere_transcribe_native/audio_exports.map +10 -0
  14. data/ext/cohere_transcribe_native/cohere_abi.cpp +1209 -0
  15. data/ext/cohere_transcribe_native/exports.macos +40 -0
  16. data/ext/cohere_transcribe_native/exports.map +45 -0
  17. data/ext/cohere_transcribe_native/extconf.rb +89 -0
  18. data/ext/cohere_transcribe_native/test/abi_smoke.rb +232 -0
  19. data/ext/cohere_transcribe_native/test/audio_matrix_smoke.cpp +121 -0
  20. data/ext/cohere_transcribe_native/test/audio_reliability_smoke.cpp +198 -0
  21. data/ext/cohere_transcribe_native/test/native_abi_reliability_smoke.cpp +186 -0
  22. data/ext/cohere_transcribe_native/test/native_batch_projection_probe.rb +81 -0
  23. data/ext/cohere_transcribe_native/test/native_cancellation_smoke.cpp +194 -0
  24. data/lib/cohere/transcribe/alignment/ATTRIBUTION.md +23 -0
  25. data/lib/cohere/transcribe/alignment/LICENSE.ctc-forced-aligner +407 -0
  26. data/lib/cohere/transcribe/alignment/LICENSE.torchaudio +25 -0
  27. data/lib/cohere/transcribe/alignment/LICENSE.uroman +26 -0
  28. data/lib/cohere/transcribe/alignment/aligner.rb +476 -0
  29. data/lib/cohere/transcribe/alignment/ctc.rb +224 -0
  30. data/lib/cohere/transcribe/alignment/text.rb +237 -0
  31. data/lib/cohere/transcribe/alignment/uroman_data.rb +4950 -0
  32. data/lib/cohere/transcribe/api.rb +173 -0
  33. data/lib/cohere/transcribe/asr/batching.rb +472 -0
  34. data/lib/cohere/transcribe/asr/failure_policy.rb +112 -0
  35. data/lib/cohere/transcribe/asr/native.rb +676 -0
  36. data/lib/cohere/transcribe/audio/ATTRIBUTION.md +8 -0
  37. data/lib/cohere/transcribe/audio/LICENSE.auditok +21 -0
  38. data/lib/cohere/transcribe/audio/decoder.rb +315 -0
  39. data/lib/cohere/transcribe/audio/ffmpeg_native.rb +248 -0
  40. data/lib/cohere/transcribe/audio/segmentation.rb +260 -0
  41. data/lib/cohere/transcribe/cli.rb +727 -0
  42. data/lib/cohere/transcribe/configuration.rb +282 -0
  43. data/lib/cohere/transcribe/constants.rb +14 -0
  44. data/lib/cohere/transcribe/dense_converter.rb +548 -0
  45. data/lib/cohere/transcribe/doctor.rb +576 -0
  46. data/lib/cohere/transcribe/errors.rb +57 -0
  47. data/lib/cohere/transcribe/gguf_writer.rb +268 -0
  48. data/lib/cohere/transcribe/hub.rb +436 -0
  49. data/lib/cohere/transcribe/input.rb +110 -0
  50. data/lib/cohere/transcribe/licenses/crispasr.txt +21 -0
  51. data/lib/cohere/transcribe/loader.rb +128 -0
  52. data/lib/cohere/transcribe/model_identity.rb +440 -0
  53. data/lib/cohere/transcribe/output/publication.rb +1118 -0
  54. data/lib/cohere/transcribe/output/rendering.rb +105 -0
  55. data/lib/cohere/transcribe/output/timing.rb +86 -0
  56. data/lib/cohere/transcribe/python_text.rb +70 -0
  57. data/lib/cohere/transcribe/pytorch_checkpoint.rb +1180 -0
  58. data/lib/cohere/transcribe/runtime/engine.rb +1676 -0
  59. data/lib/cohere/transcribe/runtime/model_provider.rb +390 -0
  60. data/lib/cohere/transcribe/runtime/precision.rb +57 -0
  61. data/lib/cohere/transcribe/runtime/preparation.rb +215 -0
  62. data/lib/cohere/transcribe/runtime/resources.rb +165 -0
  63. data/lib/cohere/transcribe/runtime/word_pipeline.rb +364 -0
  64. data/lib/cohere/transcribe/safetensors.rb +579 -0
  65. data/lib/cohere/transcribe/state/checkpoint.rb +224 -0
  66. data/lib/cohere/transcribe/state/contracts.rb +141 -0
  67. data/lib/cohere/transcribe/state/io.rb +727 -0
  68. data/lib/cohere/transcribe/state/locking.rb +211 -0
  69. data/lib/cohere/transcribe/state/manifest.rb +155 -0
  70. data/lib/cohere/transcribe/state.rb +7 -0
  71. data/lib/cohere/transcribe/types.rb +535 -0
  72. data/lib/cohere/transcribe/vad/ATTRIBUTION.md +14 -0
  73. data/lib/cohere/transcribe/vad/LICENSE.faster-whisper +21 -0
  74. data/lib/cohere/transcribe/vad/LICENSE.silero-vad +21 -0
  75. data/lib/cohere/transcribe/vad/silero.rb +344 -0
  76. data/lib/cohere/transcribe/vad/silero_vad_v6.onnx +0 -0
  77. data/lib/cohere/transcribe/vad/timestamps.rb +219 -0
  78. data/lib/cohere/transcribe/version.rb +7 -0
  79. data/lib/cohere/transcribe.rb +26 -0
  80. data/sig/cohere/transcribe.rbs +250 -0
  81. data/vendor/crispasr/AUTHORS +510 -0
  82. data/vendor/crispasr/LICENSE +21 -0
  83. data/vendor/crispasr/UPSTREAM.md +9 -0
  84. data/vendor/crispasr/VERSION +1 -0
  85. data/vendor/crispasr/ggml/AUTHORS +335 -0
  86. data/vendor/crispasr/ggml/CMakeLists.txt +512 -0
  87. data/vendor/crispasr/ggml/LICENSE +21 -0
  88. data/vendor/crispasr/ggml/README.md +49 -0
  89. data/vendor/crispasr/ggml/cmake/FindNCCL.cmake +36 -0
  90. data/vendor/crispasr/ggml/cmake/GitVars.cmake +22 -0
  91. data/vendor/crispasr/ggml/cmake/common.cmake +50 -0
  92. data/vendor/crispasr/ggml/cmake/ggml-config.cmake.in +191 -0
  93. data/vendor/crispasr/ggml/ggml.pc.in +10 -0
  94. data/vendor/crispasr/ggml/include/ggml-alloc.h +85 -0
  95. data/vendor/crispasr/ggml/include/ggml-backend.h +431 -0
  96. data/vendor/crispasr/ggml/include/ggml-blas.h +25 -0
  97. data/vendor/crispasr/ggml/include/ggml-cann.h +123 -0
  98. data/vendor/crispasr/ggml/include/ggml-cpp.h +39 -0
  99. data/vendor/crispasr/ggml/include/ggml-cpu.h +151 -0
  100. data/vendor/crispasr/ggml/include/ggml-cuda.h +50 -0
  101. data/vendor/crispasr/ggml/include/ggml-hexagon.h +19 -0
  102. data/vendor/crispasr/ggml/include/ggml-metal.h +61 -0
  103. data/vendor/crispasr/ggml/include/ggml-opencl.h +26 -0
  104. data/vendor/crispasr/ggml/include/ggml-openvino.h +37 -0
  105. data/vendor/crispasr/ggml/include/ggml-opt.h +256 -0
  106. data/vendor/crispasr/ggml/include/ggml-rpc.h +35 -0
  107. data/vendor/crispasr/ggml/include/ggml-sycl.h +49 -0
  108. data/vendor/crispasr/ggml/include/ggml-virtgpu.h +14 -0
  109. data/vendor/crispasr/ggml/include/ggml-vulkan.h +29 -0
  110. data/vendor/crispasr/ggml/include/ggml-webgpu.h +19 -0
  111. data/vendor/crispasr/ggml/include/ggml-zdnn.h +17 -0
  112. data/vendor/crispasr/ggml/include/ggml-zendnn.h +22 -0
  113. data/vendor/crispasr/ggml/include/ggml.h +2887 -0
  114. data/vendor/crispasr/ggml/include/gguf.h +204 -0
  115. data/vendor/crispasr/ggml/src/CMakeLists.txt +493 -0
  116. data/vendor/crispasr/ggml/src/ggml-alloc.c +1323 -0
  117. data/vendor/crispasr/ggml/src/ggml-backend-dl.cpp +48 -0
  118. data/vendor/crispasr/ggml/src/ggml-backend-dl.h +44 -0
  119. data/vendor/crispasr/ggml/src/ggml-backend-impl.h +275 -0
  120. data/vendor/crispasr/ggml/src/ggml-backend-meta.cpp +2145 -0
  121. data/vendor/crispasr/ggml/src/ggml-backend-reg.cpp +586 -0
  122. data/vendor/crispasr/ggml/src/ggml-backend.cpp +2437 -0
  123. data/vendor/crispasr/ggml/src/ggml-common.h +1900 -0
  124. data/vendor/crispasr/ggml/src/ggml-cpu/CMakeLists.txt +718 -0
  125. data/vendor/crispasr/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
  126. data/vendor/crispasr/ggml/src/ggml-cpu/amx/amx.h +8 -0
  127. data/vendor/crispasr/ggml/src/ggml-cpu/amx/common.h +115 -0
  128. data/vendor/crispasr/ggml/src/ggml-cpu/amx/mmq.cpp +2512 -0
  129. data/vendor/crispasr/ggml/src/ggml-cpu/amx/mmq.h +10 -0
  130. data/vendor/crispasr/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +98 -0
  131. data/vendor/crispasr/ggml/src/ggml-cpu/arch/arm/quants.c +4244 -0
  132. data/vendor/crispasr/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
  133. data/vendor/crispasr/ggml/src/ggml-cpu/arch/loongarch/quants.c +2158 -0
  134. data/vendor/crispasr/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
  135. data/vendor/crispasr/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
  136. data/vendor/crispasr/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
  137. data/vendor/crispasr/ggml/src/ggml-cpu/arch/riscv/quants.c +4455 -0
  138. data/vendor/crispasr/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
  139. data/vendor/crispasr/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
  140. data/vendor/crispasr/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
  141. data/vendor/crispasr/ggml/src/ggml-cpu/arch/wasm/quants.c +1220 -0
  142. data/vendor/crispasr/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
  143. data/vendor/crispasr/ggml/src/ggml-cpu/arch/x86/quants.c +3970 -0
  144. data/vendor/crispasr/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
  145. data/vendor/crispasr/ggml/src/ggml-cpu/arch-fallback.h +349 -0
  146. data/vendor/crispasr/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
  147. data/vendor/crispasr/ggml/src/ggml-cpu/binary-ops.h +16 -0
  148. data/vendor/crispasr/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
  149. data/vendor/crispasr/ggml/src/ggml-cpu/common.h +95 -0
  150. data/vendor/crispasr/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
  151. data/vendor/crispasr/ggml/src/ggml-cpu/ggml-cpu.c +3791 -0
  152. data/vendor/crispasr/ggml/src/ggml-cpu/ggml-cpu.cpp +703 -0
  153. data/vendor/crispasr/ggml/src/ggml-cpu/hbm.cpp +55 -0
  154. data/vendor/crispasr/ggml/src/ggml-cpu/hbm.h +8 -0
  155. data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kernels.cpp +939 -0
  156. data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kernels.h +90 -0
  157. data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1513 -0
  158. data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
  159. data/vendor/crispasr/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4051 -0
  160. data/vendor/crispasr/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
  161. data/vendor/crispasr/ggml/src/ggml-cpu/ops.cpp +11662 -0
  162. data/vendor/crispasr/ggml/src/ggml-cpu/ops.h +121 -0
  163. data/vendor/crispasr/ggml/src/ggml-cpu/quants.c +1288 -0
  164. data/vendor/crispasr/ggml/src/ggml-cpu/quants.h +103 -0
  165. data/vendor/crispasr/ggml/src/ggml-cpu/repack.cpp +4836 -0
  166. data/vendor/crispasr/ggml/src/ggml-cpu/repack.h +245 -0
  167. data/vendor/crispasr/ggml/src/ggml-cpu/simd-gemm.h +226 -0
  168. data/vendor/crispasr/ggml/src/ggml-cpu/simd-mappings.h +1329 -0
  169. data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime.cpp +1025 -0
  170. data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime.h +13 -0
  171. data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +3196 -0
  172. data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime_kernels.h +26 -0
  173. data/vendor/crispasr/ggml/src/ggml-cpu/traits.cpp +36 -0
  174. data/vendor/crispasr/ggml/src/ggml-cpu/traits.h +38 -0
  175. data/vendor/crispasr/ggml/src/ggml-cpu/unary-ops.cpp +336 -0
  176. data/vendor/crispasr/ggml/src/ggml-cpu/unary-ops.h +35 -0
  177. data/vendor/crispasr/ggml/src/ggml-cpu/vec.cpp +681 -0
  178. data/vendor/crispasr/ggml/src/ggml-cpu/vec.h +1606 -0
  179. data/vendor/crispasr/ggml/src/ggml-cuda/CMakeLists.txt +272 -0
  180. data/vendor/crispasr/ggml/src/ggml-cuda/acc.cu +61 -0
  181. data/vendor/crispasr/ggml/src/ggml-cuda/acc.cuh +5 -0
  182. data/vendor/crispasr/ggml/src/ggml-cuda/add-id.cu +58 -0
  183. data/vendor/crispasr/ggml/src/ggml-cuda/add-id.cuh +3 -0
  184. data/vendor/crispasr/ggml/src/ggml-cuda/arange.cu +34 -0
  185. data/vendor/crispasr/ggml/src/ggml-cuda/arange.cuh +5 -0
  186. data/vendor/crispasr/ggml/src/ggml-cuda/argmax.cu +91 -0
  187. data/vendor/crispasr/ggml/src/ggml-cuda/argmax.cuh +3 -0
  188. data/vendor/crispasr/ggml/src/ggml-cuda/argsort.cu +265 -0
  189. data/vendor/crispasr/ggml/src/ggml-cuda/argsort.cuh +19 -0
  190. data/vendor/crispasr/ggml/src/ggml-cuda/binbcast.cu +534 -0
  191. data/vendor/crispasr/ggml/src/ggml-cuda/binbcast.cuh +12 -0
  192. data/vendor/crispasr/ggml/src/ggml-cuda/clamp.cu +45 -0
  193. data/vendor/crispasr/ggml/src/ggml-cuda/clamp.cuh +5 -0
  194. data/vendor/crispasr/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
  195. data/vendor/crispasr/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
  196. data/vendor/crispasr/ggml/src/ggml-cuda/common.cuh +1489 -0
  197. data/vendor/crispasr/ggml/src/ggml-cuda/concat.cu +204 -0
  198. data/vendor/crispasr/ggml/src/ggml-cuda/concat.cuh +5 -0
  199. data/vendor/crispasr/ggml/src/ggml-cuda/conv-transpose-1d.cu +97 -0
  200. data/vendor/crispasr/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
  201. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
  202. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
  203. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
  204. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
  205. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d.cu +166 -0
  206. data/vendor/crispasr/ggml/src/ggml-cuda/conv2d.cuh +5 -0
  207. data/vendor/crispasr/ggml/src/ggml-cuda/convert.cu +892 -0
  208. data/vendor/crispasr/ggml/src/ggml-cuda/convert.cuh +66 -0
  209. data/vendor/crispasr/ggml/src/ggml-cuda/count-equal.cu +64 -0
  210. data/vendor/crispasr/ggml/src/ggml-cuda/count-equal.cuh +5 -0
  211. data/vendor/crispasr/ggml/src/ggml-cuda/cp-async.cuh +57 -0
  212. data/vendor/crispasr/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
  213. data/vendor/crispasr/ggml/src/ggml-cuda/cpy.cu +581 -0
  214. data/vendor/crispasr/ggml/src/ggml-cuda/cpy.cuh +7 -0
  215. data/vendor/crispasr/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
  216. data/vendor/crispasr/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
  217. data/vendor/crispasr/ggml/src/ggml-cuda/cumsum.cu +307 -0
  218. data/vendor/crispasr/ggml/src/ggml-cuda/cumsum.cuh +5 -0
  219. data/vendor/crispasr/ggml/src/ggml-cuda/dequantize.cuh +99 -0
  220. data/vendor/crispasr/ggml/src/ggml-cuda/diag.cu +77 -0
  221. data/vendor/crispasr/ggml/src/ggml-cuda/diag.cuh +5 -0
  222. data/vendor/crispasr/ggml/src/ggml-cuda/diagmask.cu +40 -0
  223. data/vendor/crispasr/ggml/src/ggml-cuda/diagmask.cuh +5 -0
  224. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-common.cuh +1212 -0
  225. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1860 -0
  226. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-tile.cu +57 -0
  227. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-tile.cuh +1309 -0
  228. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-vec.cuh +600 -0
  229. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-wmma-f16.cu +696 -0
  230. data/vendor/crispasr/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +51 -0
  231. data/vendor/crispasr/ggml/src/ggml-cuda/fattn.cu +620 -0
  232. data/vendor/crispasr/ggml/src/ggml-cuda/fattn.cuh +5 -0
  233. data/vendor/crispasr/ggml/src/ggml-cuda/fill.cu +37 -0
  234. data/vendor/crispasr/ggml/src/ggml-cuda/fill.cuh +3 -0
  235. data/vendor/crispasr/ggml/src/ggml-cuda/gated_delta_net.cu +273 -0
  236. data/vendor/crispasr/ggml/src/ggml-cuda/gated_delta_net.cuh +4 -0
  237. data/vendor/crispasr/ggml/src/ggml-cuda/getrows.cu +332 -0
  238. data/vendor/crispasr/ggml/src/ggml-cuda/getrows.cuh +15 -0
  239. data/vendor/crispasr/ggml/src/ggml-cuda/ggml-cuda.cu +5580 -0
  240. data/vendor/crispasr/ggml/src/ggml-cuda/gla.cu +93 -0
  241. data/vendor/crispasr/ggml/src/ggml-cuda/gla.cuh +3 -0
  242. data/vendor/crispasr/ggml/src/ggml-cuda/im2col.cu +274 -0
  243. data/vendor/crispasr/ggml/src/ggml-cuda/im2col.cuh +6 -0
  244. data/vendor/crispasr/ggml/src/ggml-cuda/mean.cu +75 -0
  245. data/vendor/crispasr/ggml/src/ggml-cuda/mean.cuh +3 -0
  246. data/vendor/crispasr/ggml/src/ggml-cuda/mma.cuh +1333 -0
  247. data/vendor/crispasr/ggml/src/ggml-cuda/mmf.cu +191 -0
  248. data/vendor/crispasr/ggml/src/ggml-cuda/mmf.cuh +908 -0
  249. data/vendor/crispasr/ggml/src/ggml-cuda/mmid.cu +164 -0
  250. data/vendor/crispasr/ggml/src/ggml-cuda/mmid.cuh +5 -0
  251. data/vendor/crispasr/ggml/src/ggml-cuda/mmq.cu +372 -0
  252. data/vendor/crispasr/ggml/src/ggml-cuda/mmq.cuh +4175 -0
  253. data/vendor/crispasr/ggml/src/ggml-cuda/mmvf.cu +862 -0
  254. data/vendor/crispasr/ggml/src/ggml-cuda/mmvf.cuh +14 -0
  255. data/vendor/crispasr/ggml/src/ggml-cuda/mmvq.cu +1161 -0
  256. data/vendor/crispasr/ggml/src/ggml-cuda/mmvq.cuh +16 -0
  257. data/vendor/crispasr/ggml/src/ggml-cuda/norm.cu +756 -0
  258. data/vendor/crispasr/ggml/src/ggml-cuda/norm.cuh +20 -0
  259. data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
  260. data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
  261. data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
  262. data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
  263. data/vendor/crispasr/ggml/src/ggml-cuda/out-prod.cu +68 -0
  264. data/vendor/crispasr/ggml/src/ggml-cuda/out-prod.cuh +3 -0
  265. data/vendor/crispasr/ggml/src/ggml-cuda/pad.cu +106 -0
  266. data/vendor/crispasr/ggml/src/ggml-cuda/pad.cuh +5 -0
  267. data/vendor/crispasr/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
  268. data/vendor/crispasr/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
  269. data/vendor/crispasr/ggml/src/ggml-cuda/pool2d.cu +94 -0
  270. data/vendor/crispasr/ggml/src/ggml-cuda/pool2d.cuh +5 -0
  271. data/vendor/crispasr/ggml/src/ggml-cuda/quantize.cu +443 -0
  272. data/vendor/crispasr/ggml/src/ggml-cuda/quantize.cuh +41 -0
  273. data/vendor/crispasr/ggml/src/ggml-cuda/reduce_rows.cuh +39 -0
  274. data/vendor/crispasr/ggml/src/ggml-cuda/roll.cu +67 -0
  275. data/vendor/crispasr/ggml/src/ggml-cuda/roll.cuh +5 -0
  276. data/vendor/crispasr/ggml/src/ggml-cuda/rope.cu +665 -0
  277. data/vendor/crispasr/ggml/src/ggml-cuda/rope.cuh +9 -0
  278. data/vendor/crispasr/ggml/src/ggml-cuda/scale.cu +34 -0
  279. data/vendor/crispasr/ggml/src/ggml-cuda/scale.cuh +5 -0
  280. data/vendor/crispasr/ggml/src/ggml-cuda/set-rows.cu +330 -0
  281. data/vendor/crispasr/ggml/src/ggml-cuda/set-rows.cuh +7 -0
  282. data/vendor/crispasr/ggml/src/ggml-cuda/set.cu +39 -0
  283. data/vendor/crispasr/ggml/src/ggml-cuda/set.cuh +7 -0
  284. data/vendor/crispasr/ggml/src/ggml-cuda/softcap.cu +34 -0
  285. data/vendor/crispasr/ggml/src/ggml-cuda/softcap.cuh +5 -0
  286. data/vendor/crispasr/ggml/src/ggml-cuda/softmax.cu +472 -0
  287. data/vendor/crispasr/ggml/src/ggml-cuda/softmax.cuh +7 -0
  288. data/vendor/crispasr/ggml/src/ggml-cuda/solve_tri.cu +275 -0
  289. data/vendor/crispasr/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
  290. data/vendor/crispasr/ggml/src/ggml-cuda/ssm-conv.cu +197 -0
  291. data/vendor/crispasr/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
  292. data/vendor/crispasr/ggml/src/ggml-cuda/ssm-scan.cu +342 -0
  293. data/vendor/crispasr/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
  294. data/vendor/crispasr/ggml/src/ggml-cuda/sum.cu +41 -0
  295. data/vendor/crispasr/ggml/src/ggml-cuda/sum.cuh +5 -0
  296. data/vendor/crispasr/ggml/src/ggml-cuda/sumrows.cu +43 -0
  297. data/vendor/crispasr/ggml/src/ggml-cuda/sumrows.cuh +4 -0
  298. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
  299. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
  300. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +11 -0
  301. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
  302. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
  303. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
  304. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
  305. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
  306. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
  307. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +11 -0
  308. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
  309. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
  310. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
  311. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
  312. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
  313. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +11 -0
  314. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
  315. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
  316. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
  317. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
  318. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +11 -0
  319. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
  320. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
  321. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
  322. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
  323. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
  324. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
  325. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
  326. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
  327. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
  328. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
  329. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
  330. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
  331. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
  332. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
  333. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
  334. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
  335. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
  336. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
  337. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
  338. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
  339. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
  340. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
  341. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
  342. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
  343. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
  344. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
  345. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
  346. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
  347. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
  348. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
  349. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
  350. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
  351. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
  352. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
  353. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
  354. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
  355. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
  356. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
  357. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
  358. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
  359. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
  360. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
  361. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
  362. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
  363. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
  364. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
  365. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
  366. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
  367. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
  368. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
  369. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
  370. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
  371. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
  372. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
  373. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
  374. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
  375. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
  376. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
  377. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
  378. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
  379. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
  380. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
  381. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
  382. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
  383. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
  384. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
  385. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
  386. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
  387. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
  388. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
  389. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
  390. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
  391. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
  392. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
  393. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
  394. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
  395. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
  396. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
  397. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
  398. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
  399. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
  400. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
  401. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
  402. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
  403. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
  404. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
  405. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
  406. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
  407. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
  408. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
  409. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
  410. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
  411. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
  412. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
  413. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
  414. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
  415. data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
  416. data/vendor/crispasr/ggml/src/ggml-cuda/top-k.cu +95 -0
  417. data/vendor/crispasr/ggml/src/ggml-cuda/top-k.cuh +3 -0
  418. data/vendor/crispasr/ggml/src/ggml-cuda/topk-moe.cu +415 -0
  419. data/vendor/crispasr/ggml/src/ggml-cuda/topk-moe.cuh +27 -0
  420. data/vendor/crispasr/ggml/src/ggml-cuda/tri.cu +136 -0
  421. data/vendor/crispasr/ggml/src/ggml-cuda/tri.cuh +5 -0
  422. data/vendor/crispasr/ggml/src/ggml-cuda/tsembd.cu +47 -0
  423. data/vendor/crispasr/ggml/src/ggml-cuda/tsembd.cuh +5 -0
  424. data/vendor/crispasr/ggml/src/ggml-cuda/unary.cu +644 -0
  425. data/vendor/crispasr/ggml/src/ggml-cuda/unary.cuh +116 -0
  426. data/vendor/crispasr/ggml/src/ggml-cuda/upscale.cu +293 -0
  427. data/vendor/crispasr/ggml/src/ggml-cuda/upscale.cuh +5 -0
  428. data/vendor/crispasr/ggml/src/ggml-cuda/vecdotq.cuh +1317 -0
  429. data/vendor/crispasr/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
  430. data/vendor/crispasr/ggml/src/ggml-cuda/vendors/hip.h +306 -0
  431. data/vendor/crispasr/ggml/src/ggml-cuda/vendors/musa.h +148 -0
  432. data/vendor/crispasr/ggml/src/ggml-cuda/wkv.cu +199 -0
  433. data/vendor/crispasr/ggml/src/ggml-cuda/wkv.cuh +7 -0
  434. data/vendor/crispasr/ggml/src/ggml-impl.h +783 -0
  435. data/vendor/crispasr/ggml/src/ggml-metal/CMakeLists.txt +124 -0
  436. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-common.cpp +457 -0
  437. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-common.h +52 -0
  438. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
  439. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-context.m +778 -0
  440. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-device.cpp +2117 -0
  441. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-device.h +299 -0
  442. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-device.m +2093 -0
  443. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-impl.h +1267 -0
  444. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-ops.cpp +5023 -0
  445. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-ops.h +111 -0
  446. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal.cpp +954 -0
  447. data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal.metal +11756 -0
  448. data/vendor/crispasr/ggml/src/ggml-opt.cpp +1094 -0
  449. data/vendor/crispasr/ggml/src/ggml-quants.c +5491 -0
  450. data/vendor/crispasr/ggml/src/ggml-quants.h +112 -0
  451. data/vendor/crispasr/ggml/src/ggml-threading.cpp +12 -0
  452. data/vendor/crispasr/ggml/src/ggml-threading.h +14 -0
  453. data/vendor/crispasr/ggml/src/ggml.c +7925 -0
  454. data/vendor/crispasr/ggml/src/ggml.cpp +26 -0
  455. data/vendor/crispasr/ggml/src/gguf.cpp +1556 -0
  456. data/vendor/crispasr/src/cohere-arch.h +137 -0
  457. data/vendor/crispasr/src/cohere.cpp +5642 -0
  458. data/vendor/crispasr/src/cohere.h +327 -0
  459. data/vendor/crispasr/src/cohere_batch_planner.h +82 -0
  460. data/vendor/crispasr/src/cohere_chunking.h +64 -0
  461. data/vendor/crispasr/src/cohere_decoder_batch_layout.h +60 -0
  462. data/vendor/crispasr/src/cohere_encoder_padded_layout.h +27 -0
  463. data/vendor/crispasr/src/cohere_frontend.cpp +189 -0
  464. data/vendor/crispasr/src/cohere_frontend.h +31 -0
  465. data/vendor/crispasr/src/cohere_ragged_controller.h +149 -0
  466. data/vendor/crispasr/src/cohere_token_renderer.h +181 -0
  467. data/vendor/crispasr/src/core/attention.h +924 -0
  468. data/vendor/crispasr/src/core/audio_chunking.h +97 -0
  469. data/vendor/crispasr/src/core/beam_decode.h +486 -0
  470. data/vendor/crispasr/src/core/cpu_ops.h +135 -0
  471. data/vendor/crispasr/src/core/gguf_loader.cpp +1021 -0
  472. data/vendor/crispasr/src/core/gguf_loader.h +216 -0
  473. data/vendor/crispasr/src/core/gpu_backend_pref.h +119 -0
  474. data/vendor/crispasr/src/core/mel.cpp +519 -0
  475. data/vendor/crispasr/src/core/mel.h +265 -0
  476. data/vendor/crispasr/src/core/ngram_loop_fix.h +173 -0
  477. data/vendor/crispasr/src/core/repetition_loop_guard.h +54 -0
  478. data/vendor/crispasr/src/crispasr_imatrix.cpp +255 -0
  479. data/vendor/crispasr/src/crispasr_imatrix.h +38 -0
  480. metadata +596 -0
@@ -0,0 +1,727 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+ require "fiddle"
5
+ require "json"
6
+ require "securerandom"
7
+
8
+ module Cohere
9
+ module Transcribe
10
+ module State
11
+ STATE_SCHEMA_VERSION = 1
12
+ STATE_SUFFIX = ".cohere-transcribe.manifest.json"
13
+ CHECKPOINT_SUFFIX = ".cohere-transcribe.asr.json"
14
+ MAX_STATE_BYTES = 64 * 1024 * 1024
15
+ DEFERRED_PUBLICATION_EXCEPTIONS = {
16
+ SignalException => :never,
17
+ SystemExit => :never
18
+ }.freeze
19
+
20
+ DirectoryBinding = Data.define(
21
+ :access_path,
22
+ :canonical_path,
23
+ :device,
24
+ :inode
25
+ ) do
26
+ def self.capture(path)
27
+ access_path = Pathname(path).expand_path.cleanpath
28
+ canonical_path = access_path.realpath
29
+ stat = canonical_path.lstat
30
+ unless stat.directory? && !stat.symlink?
31
+ raise TranscriptionRuntimeError, "Publication parent is not a real directory: #{access_path}"
32
+ end
33
+
34
+ new(
35
+ access_path: access_path.freeze,
36
+ canonical_path: canonical_path.freeze,
37
+ device: stat.dev,
38
+ inode: stat.ino
39
+ )
40
+ rescue SystemCallError, ArgumentError => e
41
+ raise TranscriptionRuntimeError,
42
+ "Cannot bind publication parent #{path}: #{e.message}"
43
+ end
44
+
45
+ def verify!
46
+ resolved = access_path.realpath
47
+ stat = canonical_path.lstat
48
+ valid = resolved == canonical_path && stat.directory? && !stat.symlink? &&
49
+ stat.dev == device && stat.ino == inode
50
+ return self if valid
51
+
52
+ raise TranscriptionRuntimeError,
53
+ "Publication parent changed after planning: #{access_path}"
54
+ rescue SystemCallError, ArgumentError => e
55
+ raise TranscriptionRuntimeError,
56
+ "Publication parent changed after planning: #{access_path} (#{e.message})"
57
+ end
58
+ end
59
+
60
+ # A retained directory descriptor is the authority for publication I/O.
61
+ # Path checks alone cannot close a rename/symlink TOCTOU window: after
62
+ # this object opens and verifies the planned inode, every entry operation
63
+ # is performed with the POSIX *at family against that descriptor.
64
+ class BoundDirectory
65
+ AT_FUNCTION_SIGNATURES = {
66
+ openat: [[Fiddle::TYPE_INT, Fiddle::TYPE_VOIDP, Fiddle::TYPE_INT, Fiddle::TYPE_INT], Fiddle::TYPE_INT],
67
+ renameat: [
68
+ [Fiddle::TYPE_INT, Fiddle::TYPE_VOIDP, Fiddle::TYPE_INT, Fiddle::TYPE_VOIDP],
69
+ Fiddle::TYPE_INT
70
+ ],
71
+ unlinkat: [[Fiddle::TYPE_INT, Fiddle::TYPE_VOIDP, Fiddle::TYPE_INT], Fiddle::TYPE_INT],
72
+ mkdirat: [[Fiddle::TYPE_INT, Fiddle::TYPE_VOIDP, Fiddle::TYPE_INT], Fiddle::TYPE_INT]
73
+ }.freeze
74
+
75
+ class << self
76
+ def open(binding, guards: [binding])
77
+ ensure_supported!
78
+ succeeded = false
79
+ guards = guards.uniq.freeze
80
+ guards.each(&:verify!)
81
+ flags = File::RDONLY | File::NONBLOCK | File.const_get(:NOFOLLOW)
82
+ descriptor = IO.sysopen(binding.canonical_path.to_s, flags)
83
+ handle = File.new(descriptor, "rb", autoclose: true)
84
+ descriptor = nil
85
+ handle.close_on_exec = true
86
+ stat = handle.stat
87
+ unless stat.directory? && stat.dev == binding.device && stat.ino == binding.inode
88
+ raise TranscriptionRuntimeError,
89
+ "Publication parent changed while it was being opened: #{binding.access_path}"
90
+ end
91
+ guards.each(&:verify!)
92
+ bound = new(binding, handle, guards)
93
+ succeeded = true
94
+ bound
95
+ rescue Interrupt, SystemExit
96
+ raise
97
+ rescue SystemCallError, ArgumentError => e
98
+ raise TranscriptionRuntimeError,
99
+ "Cannot open planned publication parent #{binding.access_path}: #{e.message}"
100
+ ensure
101
+ handle&.close if handle && !succeeded
102
+ IO.new(descriptor).close if descriptor
103
+ end
104
+
105
+ def ensure_supported!
106
+ required_constants = %i[NOFOLLOW]
107
+ missing_constants = required_constants.reject { |name| File.const_defined?(name) }
108
+ missing_functions = AT_FUNCTION_SIGNATURES.keys.reject do |name|
109
+ function(name)
110
+ true
111
+ rescue Fiddle::DLError
112
+ false
113
+ end
114
+ return if missing_constants.empty? && missing_functions.empty?
115
+
116
+ details = (missing_constants + missing_functions).join(", ")
117
+ raise TranscriptionRuntimeError,
118
+ "Descriptor-relative publication is unavailable on this platform (missing #{details})"
119
+ end
120
+
121
+ def function(name)
122
+ @function_guard ||= Mutex.new
123
+ @functions ||= {}
124
+ @function_guard.synchronize do
125
+ @functions[name] ||= begin
126
+ arguments, result = AT_FUNCTION_SIGNATURES.fetch(name)
127
+ address = Fiddle::Handle::DEFAULT[name.to_s]
128
+ Fiddle::Function.new(address, arguments, result)
129
+ end
130
+ end
131
+ end
132
+ end
133
+
134
+ attr_reader :binding
135
+
136
+ def initialize(binding, handle, guards)
137
+ @binding = binding
138
+ @handle = handle
139
+ @guards = guards
140
+ @closed = false
141
+ end
142
+
143
+ def create_temporary(basename, suffix)
144
+ validate_entry_name!(basename)
145
+ 100.times do
146
+ name = ".#{basename}.#{SecureRandom.hex(12)}#{suffix}"
147
+ begin
148
+ handle = open_entry(
149
+ name,
150
+ File::WRONLY | File::CREAT | File::EXCL | File::NOFOLLOW,
151
+ 0o600,
152
+ mode: "wb"
153
+ )
154
+ return [name.freeze, handle]
155
+ rescue Errno::EEXIST
156
+ next
157
+ end
158
+ end
159
+ raise TranscriptionRuntimeError,
160
+ "Cannot allocate a unique publication temporary for #{basename}"
161
+ end
162
+
163
+ def open_regular(name, writable: false)
164
+ flags = writable ? File::RDWR : File::RDONLY
165
+ flags |= File::NOFOLLOW | File::NONBLOCK
166
+ handle = open_entry(name, flags, 0, mode: writable ? "r+b" : "rb")
167
+ unless handle.stat.file?
168
+ handle.close
169
+ raise TranscriptionRuntimeError,
170
+ "Publication entry is not a regular file: #{display_path(name)}"
171
+ end
172
+ handle
173
+ rescue Errno::ELOOP, Errno::EISDIR, Errno::ENXIO => e
174
+ raise TranscriptionRuntimeError,
175
+ "Publication entry is not a regular file: #{display_path(name)}",
176
+ cause: e
177
+ end
178
+
179
+ def regular_entry?(name)
180
+ handle = nil
181
+ Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
182
+ handle = open_regular(name)
183
+ end
184
+ true
185
+ rescue Errno::ENOENT
186
+ false
187
+ ensure
188
+ handle&.close
189
+ end
190
+
191
+ def same_regular_entry?(name, expected_stat)
192
+ handle = nil
193
+ Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
194
+ handle = open_regular(name)
195
+ end
196
+ current = handle.stat
197
+ current.dev == expected_stat.dev && current.ino == expected_stat.ino
198
+ rescue Errno::ENOENT
199
+ false
200
+ ensure
201
+ handle&.close
202
+ end
203
+
204
+ def rename(source, destination)
205
+ validate_entry_name!(source)
206
+ validate_entry_name!(destination)
207
+ call_at!(:renameat, descriptor, source, descriptor, destination)
208
+ nil
209
+ end
210
+
211
+ def mkdir(name, mode = 0o777)
212
+ validate_entry_name!(name)
213
+ call_at!(:mkdirat, descriptor, name, mode)
214
+ nil
215
+ rescue Errno::EEXIST
216
+ nil
217
+ end
218
+
219
+ def open_child_directory(name, access_path:, canonical_path:)
220
+ validate_entry_name!(name)
221
+ handle = open_entry(
222
+ name,
223
+ File::RDONLY | File::NONBLOCK | File::NOFOLLOW,
224
+ 0,
225
+ mode: "rb"
226
+ )
227
+ stat = handle.stat
228
+ unless stat.directory?
229
+ raise TranscriptionRuntimeError,
230
+ "Publication directory component is not a real directory: #{access_path}"
231
+ end
232
+ child_binding = DirectoryBinding.new(
233
+ access_path: Pathname(access_path).freeze,
234
+ canonical_path: Pathname(canonical_path).freeze,
235
+ device: stat.dev,
236
+ inode: stat.ino
237
+ )
238
+ child = self.class.new(child_binding, handle, (@guards + [child_binding]).uniq.freeze)
239
+ child.verify!
240
+ handle = nil
241
+ child
242
+ rescue Errno::ELOOP, Errno::EISDIR, Errno::ENXIO => e
243
+ raise TranscriptionRuntimeError,
244
+ "Publication directory component is not a real directory: #{access_path}",
245
+ cause: e
246
+ ensure
247
+ handle&.close
248
+ end
249
+
250
+ def unlink(name, missing_ok: false)
251
+ validate_entry_name!(name)
252
+ call_at!(:unlinkat, descriptor, name, 0)
253
+ nil
254
+ rescue Errno::ENOENT
255
+ raise unless missing_ok
256
+
257
+ nil
258
+ end
259
+
260
+ def verify!
261
+ stat = @handle.stat
262
+ unless stat.directory? && stat.dev == binding.device && stat.ino == binding.inode
263
+ raise TranscriptionRuntimeError,
264
+ "Retained publication parent changed identity: #{binding.access_path}"
265
+ end
266
+ @guards.each(&:verify!)
267
+ self
268
+ end
269
+
270
+ def fsync
271
+ @handle.fsync
272
+ rescue Errno::EACCES, Errno::EBADF, Errno::EINVAL, Errno::EISDIR,
273
+ Errno::ENOTSUP, Errno::EPERM
274
+ nil
275
+ end
276
+
277
+ def close
278
+ return if @closed
279
+
280
+ @handle.close
281
+ @closed = true
282
+ nil
283
+ end
284
+
285
+ def display_path(name)
286
+ binding.canonical_path.join(name)
287
+ end
288
+
289
+ private
290
+
291
+ def descriptor
292
+ @handle.fileno
293
+ end
294
+
295
+ def open_entry(name, flags, permissions, mode:)
296
+ validate_entry_name!(name)
297
+ file_descriptor = call_at!(:openat, descriptor, name, flags, permissions)
298
+ handle = File.new(file_descriptor, mode, autoclose: true)
299
+ file_descriptor = nil
300
+ handle.close_on_exec = true
301
+ handle
302
+ ensure
303
+ IO.new(file_descriptor).close if file_descriptor
304
+ end
305
+
306
+ def call_at!(name, *)
307
+ result = self.class.function(name).call(*)
308
+ return result unless result == -1
309
+
310
+ error_number = Fiddle.last_error
311
+ raise SystemCallError.new("#{name} #{binding.canonical_path}", error_number)
312
+ end
313
+
314
+ def validate_entry_name!(name)
315
+ value = name.to_s
316
+ return if !value.empty? && value != "." && value != ".." &&
317
+ !value.include?(File::SEPARATOR) && !value.include?("\0")
318
+
319
+ raise ArgumentError, "Invalid descriptor-relative publication name: #{value.inspect}"
320
+ end
321
+ end
322
+
323
+ SourceSnapshot = Data.define(
324
+ :canonical_path,
325
+ :device,
326
+ :inode,
327
+ :size,
328
+ :mtime_ns,
329
+ :ctime_ns
330
+ ) do
331
+ def self.capture(path)
332
+ canonical = Pathname(path).realpath
333
+ stat = canonical.stat
334
+ new(
335
+ canonical_path: canonical.to_s.freeze,
336
+ device: stat.dev,
337
+ inode: stat.ino,
338
+ size: stat.size,
339
+ mtime_ns: State.time_nanoseconds(stat.mtime),
340
+ ctime_ns: State.time_nanoseconds(stat.ctime)
341
+ )
342
+ end
343
+
344
+ def payload
345
+ {
346
+ "canonical_path" => canonical_path,
347
+ "snapshot" => {
348
+ "device" => device,
349
+ "inode" => inode,
350
+ "size" => size,
351
+ "mtime_ns" => mtime_ns,
352
+ "ctime_ns" => ctime_ns
353
+ }
354
+ }
355
+ end
356
+ end
357
+
358
+ module_function
359
+
360
+ def with_bound_parent(path, directory_binding: nil, guard_bindings: nil)
361
+ path = Pathname(path).expand_path
362
+ directory_binding ||= DirectoryBinding.capture(path.dirname)
363
+ unless [directory_binding.access_path, directory_binding.canonical_path].include?(path.dirname)
364
+ raise TranscriptionRuntimeError,
365
+ "Publication path is outside its planned parent: #{path}"
366
+ end
367
+ guards = Array(guard_bindings).compact
368
+ guards << directory_binding
369
+ bound = nil
370
+ Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
371
+ bound = BoundDirectory.open(directory_binding, guards: guards.uniq)
372
+ end
373
+ yield bound, path.basename.to_s
374
+ ensure
375
+ bound&.close
376
+ end
377
+
378
+ def ensure_bound_directory(path, root_binding: nil, operation_hook: nil)
379
+ target = Pathname(path).expand_path.cleanpath
380
+ bindings = []
381
+ current = nil
382
+ if root_binding
383
+ root_binding.verify!
384
+ relative = target.relative_path_from(root_binding.canonical_path)
385
+ if relative.each_filename.first == ".."
386
+ raise TranscriptionRuntimeError,
387
+ "Publication directory escapes its bound root: #{target}"
388
+ end
389
+ current_binding = root_binding
390
+ access_cursor = root_binding.canonical_path
391
+ canonical_cursor = root_binding.canonical_path
392
+ components = relative.each_filename.reject { |name| name == "." }
393
+ else
394
+ missing = []
395
+ cursor = target
396
+ until cursor.exist? || cursor.symlink?
397
+ parent = cursor.dirname
398
+ raise TranscriptionRuntimeError, "Cannot resolve publication directory #{target}" if parent == cursor
399
+
400
+ missing.unshift(cursor.basename.to_s)
401
+ cursor = parent
402
+ end
403
+ current_binding = DirectoryBinding.capture(cursor)
404
+ access_cursor = current_binding.access_path
405
+ canonical_cursor = current_binding.canonical_path
406
+ components = missing
407
+ end
408
+
409
+ Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
410
+ current = BoundDirectory.open(current_binding, guards: [current_binding])
411
+ bindings << current
412
+ end
413
+ operation_hook&.call(:ancestor_opened, current.binding.access_path)
414
+ components.each do |component|
415
+ operation_hook&.call(:before_mkdir, access_cursor.join(component))
416
+ current.mkdir(component)
417
+ operation_hook&.call(:after_mkdir, access_cursor.join(component))
418
+ access_cursor = access_cursor.join(component)
419
+ canonical_cursor = canonical_cursor.join(component)
420
+ Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
421
+ current = current.open_child_directory(
422
+ component,
423
+ access_path: access_cursor,
424
+ canonical_path: canonical_cursor
425
+ )
426
+ bindings << current
427
+ end
428
+ end
429
+ bindings.each(&:verify!)
430
+ current.binding
431
+ rescue ArgumentError => e
432
+ raise TranscriptionRuntimeError, "Cannot bind publication directory #{path}: #{e.message}"
433
+ ensure
434
+ bindings&.reverse_each(&:close)
435
+ end
436
+
437
+ def time_nanoseconds(time)
438
+ (time.to_i * 1_000_000_000) + time.nsec
439
+ end
440
+
441
+ def output_parent_and_stem(output_paths)
442
+ raise ArgumentError, "An output set must contain at least one path" if output_paths.empty?
443
+
444
+ paths = output_paths.values.map { |path| Pathname(path) }
445
+ parent = paths.first.dirname
446
+ stem = paths.first.basename(paths.first.extname).to_s
447
+ unless paths.drop(1).all? do |path|
448
+ path.dirname == parent && path.basename(path.extname).to_s == stem
449
+ end
450
+ raise ArgumentError, "All formats in one output set must share a parent and stem"
451
+ end
452
+
453
+ [parent, stem]
454
+ end
455
+
456
+ def state_path_for_outputs(output_paths)
457
+ parent, stem = output_parent_and_stem(output_paths)
458
+ parent.join(".#{stem}#{STATE_SUFFIX}")
459
+ end
460
+
461
+ def checkpoint_path_for_outputs(output_paths)
462
+ parent, stem = output_parent_and_stem(output_paths)
463
+ parent.join(".#{stem}#{CHECKPOINT_SUFFIX}")
464
+ end
465
+
466
+ def canonical_json(value)
467
+ JSON.generate(deep_sort(value))
468
+ end
469
+
470
+ def envelope(payload)
471
+ {
472
+ "schema_version" => STATE_SCHEMA_VERSION,
473
+ "payload_sha256" => Digest::SHA256.hexdigest(canonical_json(payload)),
474
+ "payload" => payload
475
+ }
476
+ end
477
+
478
+ def decode_state(path, directory_binding: nil, guard_bindings: nil)
479
+ path = Pathname(path)
480
+ contents = securely_read_regular_file(
481
+ path,
482
+ directory_binding: directory_binding,
483
+ guard_bindings: guard_bindings
484
+ )
485
+ decoded = JSON.parse(contents)
486
+ return [nil, "state marker root is not an object"] unless decoded.is_a?(Hash)
487
+ return [nil, "state marker schema is unsupported"] unless decoded["schema_version"] == STATE_SCHEMA_VERSION
488
+
489
+ payload = decoded["payload"]
490
+ return [nil, "state marker payload is not an object"] unless payload.is_a?(Hash)
491
+
492
+ expected = Digest::SHA256.hexdigest(canonical_json(payload))
493
+ return [nil, "state marker integrity check failed"] unless secure_digest_equal?(decoded["payload_sha256"], expected)
494
+
495
+ [payload, nil]
496
+ rescue Errno::ENOENT
497
+ [nil, "state marker is missing or not a regular file"]
498
+ rescue JSON::ParserError, JSON::GeneratorError, EncodingError, SystemCallError,
499
+ TranscriptionRuntimeError, TypeError, ArgumentError => e
500
+ [nil, "state marker is unreadable (#{e.class}: #{e.message})"]
501
+ end
502
+
503
+ def write_state_atomic(path, payload, source_snapshot:, directory_binding: nil,
504
+ guard_bindings: nil, operation_hook: nil, rename: nil)
505
+ path = Pathname(path).expand_path
506
+ directory_binding ||= ensure_bound_directory(path.dirname)
507
+ with_bound_parent(
508
+ path,
509
+ directory_binding: directory_binding,
510
+ guard_bindings: guard_bindings
511
+ ) do |bound, basename|
512
+ operation_hook&.call(:directory_opened, path)
513
+ temporary_name = nil
514
+ temporary = nil
515
+ backup_name = nil
516
+ backup = nil
517
+ source = nil
518
+ committed = false
519
+ preserved_backup = false
520
+ primary_failed = false
521
+ begin
522
+ mode = bound_state_mode(bound, basename)
523
+ Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
524
+ temporary_name, temporary = bound.create_temporary(basename, ".tmp")
525
+ end
526
+ temporary.chmod(mode)
527
+ operation_hook&.call(:before_stage_write, path)
528
+ temporary.write(JSON.pretty_generate(envelope(payload)))
529
+ temporary.write("\n")
530
+ temporary.flush
531
+ temporary.fsync
532
+ temporary.close
533
+ temporary = nil
534
+ bound.verify!
535
+
536
+ operation_hook&.call(:before_backup_open, path)
537
+ Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
538
+ source = begin
539
+ bound.open_regular(basename)
540
+ rescue Errno::ENOENT
541
+ nil
542
+ end
543
+ end
544
+ if source
545
+ Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
546
+ backup_name, backup = bound.create_temporary(basename, ".bak")
547
+ end
548
+ begin
549
+ backup.chmod(source.stat.mode & 0o7777)
550
+ operation_hook&.call(:before_backup_copy, path)
551
+ IO.copy_stream(source, backup)
552
+ backup.flush
553
+ backup.fsync
554
+ ensure
555
+ source.close
556
+ backup.close
557
+ end
558
+ end
559
+
560
+ ensure_source_unchanged!(source_snapshot)
561
+ bound.verify!
562
+ operation_hook&.call(:before_rename, path)
563
+ Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
564
+ if rename
565
+ committed = true
566
+ rename.call(bound, temporary_name, basename, :commit)
567
+ else
568
+ bound.rename(temporary_name, basename)
569
+ committed = true
570
+ end
571
+ end
572
+ operation_hook&.call(:after_rename, path)
573
+ bound.verify!
574
+ bound.fsync
575
+ path
576
+ rescue Exception => e # rubocop:disable Lint/RescueException -- rollback includes interrupts
577
+ primary_failed = true
578
+ if committed
579
+ begin
580
+ Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
581
+ bound.unlink(basename, missing_ok: true)
582
+ if backup_name
583
+ if rename
584
+ rename.call(bound, backup_name, basename, :rollback)
585
+ else
586
+ bound.rename(backup_name, basename)
587
+ end
588
+ end
589
+ backup_name = nil
590
+ end
591
+ bound.fsync
592
+ rescue SystemCallError, TranscriptionRuntimeError => rollback_error
593
+ retained = backup_name ? bound.display_path(backup_name) : nil
594
+ preserved_backup = !backup_name.nil?
595
+ raise TranscriptionRuntimeError,
596
+ "State commit failed and rollback was incomplete (#{rollback_error.message}); " \
597
+ "preserved backup: #{retained}",
598
+ cause: e
599
+ end
600
+ end
601
+ raise e
602
+ ensure
603
+ cleanup_failure = nil
604
+ cleanup_operations = [
605
+ -> { temporary&.close unless temporary&.closed? },
606
+ -> { backup&.close unless backup&.closed? },
607
+ -> { bound.unlink(temporary_name, missing_ok: true) if temporary_name },
608
+ lambda do
609
+ bound.unlink(backup_name, missing_ok: true) if backup_name && !preserved_backup
610
+ end
611
+ ]
612
+ cleanup_operations.each do |operation|
613
+ operation.call
614
+ rescue SystemCallError, IOError => e
615
+ cleanup_failure ||= e
616
+ end
617
+ raise cleanup_failure if cleanup_failure && !primary_failed
618
+ end
619
+ end
620
+ end
621
+
622
+ def bound_state_mode(bound, basename)
623
+ existing = nil
624
+ Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
625
+ existing = bound.open_regular(basename)
626
+ end
627
+ existing.stat.mode & 0o7777
628
+ rescue Errno::ENOENT
629
+ current_umask = File.umask
630
+ File.umask(current_umask)
631
+ 0o666 & ~current_umask
632
+ ensure
633
+ existing&.close
634
+ end
635
+ private_class_method :bound_state_mode
636
+
637
+ def ensure_source_unchanged!(snapshot)
638
+ return if SourceSnapshot.capture(snapshot.canonical_path) == snapshot
639
+
640
+ raise TranscriptionRuntimeError, "Source changed while processing: #{snapshot.canonical_path}"
641
+ rescue SystemCallError, ArgumentError => e
642
+ raise TranscriptionRuntimeError,
643
+ "Cannot re-check source #{snapshot.canonical_path}: #{e.message}"
644
+ end
645
+
646
+ def securely_read_regular_file(path, directory_binding: nil, guard_bindings: nil)
647
+ if directory_binding
648
+ return with_bound_parent(
649
+ path,
650
+ directory_binding: directory_binding,
651
+ guard_bindings: guard_bindings
652
+ ) do |bound, basename|
653
+ bound.verify!
654
+ handle = nil
655
+ Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
656
+ handle = bound.open_regular(basename)
657
+ end
658
+ opened = handle.stat
659
+ raise TranscriptionRuntimeError, "State marker is too large: #{path}" if opened.size > MAX_STATE_BYTES
660
+
661
+ contents = handle.read(MAX_STATE_BYTES + 1)
662
+ raise TranscriptionRuntimeError, "State marker is too large: #{path}" if contents.bytesize > MAX_STATE_BYTES
663
+ unless bound.same_regular_entry?(basename, opened)
664
+ raise TranscriptionRuntimeError, "State marker changed while it was being read: #{path}"
665
+ end
666
+
667
+ bound.verify!
668
+ contents
669
+ ensure
670
+ handle&.close
671
+ end
672
+ end
673
+
674
+ before = path.lstat
675
+ raise TranscriptionRuntimeError, "State marker is not a regular file: #{path}" unless before.file? && !before.symlink?
676
+ raise TranscriptionRuntimeError, "State marker is too large: #{path}" if before.size > MAX_STATE_BYTES
677
+
678
+ flags = File::RDONLY
679
+ flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
680
+ flags |= File::CLOEXEC if defined?(File::CLOEXEC)
681
+ descriptor = ::IO.sysopen(path.to_s, flags)
682
+ handle = ::IO.new(descriptor, "rb", autoclose: true)
683
+ descriptor = nil
684
+ opened = handle.stat
685
+ current = path.lstat
686
+ unless opened.file? && same_file?(opened, before) && same_file?(opened, current)
687
+ raise TranscriptionRuntimeError, "State marker changed while it was being opened: #{path}"
688
+ end
689
+
690
+ handle.read(MAX_STATE_BYTES + 1).tap do |contents|
691
+ raise TranscriptionRuntimeError, "State marker is too large: #{path}" if contents.bytesize > MAX_STATE_BYTES
692
+ end
693
+ ensure
694
+ handle&.close
695
+ ::IO.new(descriptor).close if descriptor
696
+ end
697
+
698
+ def same_file?(first, second)
699
+ first.dev == second.dev && first.ino == second.ino && first.file? && second.file?
700
+ end
701
+
702
+ def secure_digest_equal?(actual, expected)
703
+ return false unless actual.is_a?(String) && actual.bytesize == expected.bytesize
704
+
705
+ difference = 0
706
+ actual.bytes.zip(expected.bytes) { |left, right| difference |= left ^ right }
707
+ difference.zero?
708
+ end
709
+
710
+ def deep_sort(value)
711
+ case value
712
+ when Hash
713
+ value.keys.map(&:to_s).sort.to_h do |key|
714
+ original_key = value.key?(key) ? key : value.keys.find { |candidate| candidate.to_s == key }
715
+ [key, deep_sort(value.fetch(original_key))]
716
+ end
717
+ when Array
718
+ value.map { |item| deep_sort(item) }
719
+ when Pathname
720
+ value.to_s
721
+ else
722
+ value
723
+ end
724
+ end
725
+ end
726
+ end
727
+ end