cohere-transcribe 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +21 -0
- data/LICENSE.txt +201 -0
- data/NOTICE +5 -0
- data/README.md +265 -0
- data/THIRD_PARTY_NOTICES.md +67 -0
- data/exe/cohere-transcribe +8 -0
- data/exe/cohere-transcribe-doctor +8 -0
- data/ext/cohere_transcribe_native/CMakeLists.txt +262 -0
- data/ext/cohere_transcribe_native/README.md +26 -0
- data/ext/cohere_transcribe_native/audio_abi.cpp +1416 -0
- data/ext/cohere_transcribe_native/audio_exports.macos +5 -0
- data/ext/cohere_transcribe_native/audio_exports.map +10 -0
- data/ext/cohere_transcribe_native/cohere_abi.cpp +1209 -0
- data/ext/cohere_transcribe_native/exports.macos +40 -0
- data/ext/cohere_transcribe_native/exports.map +45 -0
- data/ext/cohere_transcribe_native/extconf.rb +89 -0
- data/ext/cohere_transcribe_native/test/abi_smoke.rb +232 -0
- data/ext/cohere_transcribe_native/test/audio_matrix_smoke.cpp +121 -0
- data/ext/cohere_transcribe_native/test/audio_reliability_smoke.cpp +198 -0
- data/ext/cohere_transcribe_native/test/native_abi_reliability_smoke.cpp +186 -0
- data/ext/cohere_transcribe_native/test/native_batch_projection_probe.rb +81 -0
- data/ext/cohere_transcribe_native/test/native_cancellation_smoke.cpp +194 -0
- data/lib/cohere/transcribe/alignment/ATTRIBUTION.md +23 -0
- data/lib/cohere/transcribe/alignment/LICENSE.ctc-forced-aligner +407 -0
- data/lib/cohere/transcribe/alignment/LICENSE.torchaudio +25 -0
- data/lib/cohere/transcribe/alignment/LICENSE.uroman +26 -0
- data/lib/cohere/transcribe/alignment/aligner.rb +476 -0
- data/lib/cohere/transcribe/alignment/ctc.rb +224 -0
- data/lib/cohere/transcribe/alignment/text.rb +237 -0
- data/lib/cohere/transcribe/alignment/uroman_data.rb +4950 -0
- data/lib/cohere/transcribe/api.rb +173 -0
- data/lib/cohere/transcribe/asr/batching.rb +472 -0
- data/lib/cohere/transcribe/asr/failure_policy.rb +112 -0
- data/lib/cohere/transcribe/asr/native.rb +676 -0
- data/lib/cohere/transcribe/audio/ATTRIBUTION.md +8 -0
- data/lib/cohere/transcribe/audio/LICENSE.auditok +21 -0
- data/lib/cohere/transcribe/audio/decoder.rb +315 -0
- data/lib/cohere/transcribe/audio/ffmpeg_native.rb +248 -0
- data/lib/cohere/transcribe/audio/segmentation.rb +260 -0
- data/lib/cohere/transcribe/cli.rb +727 -0
- data/lib/cohere/transcribe/configuration.rb +282 -0
- data/lib/cohere/transcribe/constants.rb +14 -0
- data/lib/cohere/transcribe/dense_converter.rb +548 -0
- data/lib/cohere/transcribe/doctor.rb +576 -0
- data/lib/cohere/transcribe/errors.rb +57 -0
- data/lib/cohere/transcribe/gguf_writer.rb +268 -0
- data/lib/cohere/transcribe/hub.rb +436 -0
- data/lib/cohere/transcribe/input.rb +110 -0
- data/lib/cohere/transcribe/licenses/crispasr.txt +21 -0
- data/lib/cohere/transcribe/loader.rb +128 -0
- data/lib/cohere/transcribe/model_identity.rb +440 -0
- data/lib/cohere/transcribe/output/publication.rb +1118 -0
- data/lib/cohere/transcribe/output/rendering.rb +105 -0
- data/lib/cohere/transcribe/output/timing.rb +86 -0
- data/lib/cohere/transcribe/python_text.rb +70 -0
- data/lib/cohere/transcribe/pytorch_checkpoint.rb +1180 -0
- data/lib/cohere/transcribe/runtime/engine.rb +1676 -0
- data/lib/cohere/transcribe/runtime/model_provider.rb +390 -0
- data/lib/cohere/transcribe/runtime/precision.rb +57 -0
- data/lib/cohere/transcribe/runtime/preparation.rb +215 -0
- data/lib/cohere/transcribe/runtime/resources.rb +165 -0
- data/lib/cohere/transcribe/runtime/word_pipeline.rb +364 -0
- data/lib/cohere/transcribe/safetensors.rb +579 -0
- data/lib/cohere/transcribe/state/checkpoint.rb +224 -0
- data/lib/cohere/transcribe/state/contracts.rb +141 -0
- data/lib/cohere/transcribe/state/io.rb +727 -0
- data/lib/cohere/transcribe/state/locking.rb +211 -0
- data/lib/cohere/transcribe/state/manifest.rb +155 -0
- data/lib/cohere/transcribe/state.rb +7 -0
- data/lib/cohere/transcribe/types.rb +535 -0
- data/lib/cohere/transcribe/vad/ATTRIBUTION.md +14 -0
- data/lib/cohere/transcribe/vad/LICENSE.faster-whisper +21 -0
- data/lib/cohere/transcribe/vad/LICENSE.silero-vad +21 -0
- data/lib/cohere/transcribe/vad/silero.rb +344 -0
- data/lib/cohere/transcribe/vad/silero_vad_v6.onnx +0 -0
- data/lib/cohere/transcribe/vad/timestamps.rb +219 -0
- data/lib/cohere/transcribe/version.rb +7 -0
- data/lib/cohere/transcribe.rb +26 -0
- data/sig/cohere/transcribe.rbs +250 -0
- data/vendor/crispasr/AUTHORS +510 -0
- data/vendor/crispasr/LICENSE +21 -0
- data/vendor/crispasr/UPSTREAM.md +9 -0
- data/vendor/crispasr/VERSION +1 -0
- data/vendor/crispasr/ggml/AUTHORS +335 -0
- data/vendor/crispasr/ggml/CMakeLists.txt +512 -0
- data/vendor/crispasr/ggml/LICENSE +21 -0
- data/vendor/crispasr/ggml/README.md +49 -0
- data/vendor/crispasr/ggml/cmake/FindNCCL.cmake +36 -0
- data/vendor/crispasr/ggml/cmake/GitVars.cmake +22 -0
- data/vendor/crispasr/ggml/cmake/common.cmake +50 -0
- data/vendor/crispasr/ggml/cmake/ggml-config.cmake.in +191 -0
- data/vendor/crispasr/ggml/ggml.pc.in +10 -0
- data/vendor/crispasr/ggml/include/ggml-alloc.h +85 -0
- data/vendor/crispasr/ggml/include/ggml-backend.h +431 -0
- data/vendor/crispasr/ggml/include/ggml-blas.h +25 -0
- data/vendor/crispasr/ggml/include/ggml-cann.h +123 -0
- data/vendor/crispasr/ggml/include/ggml-cpp.h +39 -0
- data/vendor/crispasr/ggml/include/ggml-cpu.h +151 -0
- data/vendor/crispasr/ggml/include/ggml-cuda.h +50 -0
- data/vendor/crispasr/ggml/include/ggml-hexagon.h +19 -0
- data/vendor/crispasr/ggml/include/ggml-metal.h +61 -0
- data/vendor/crispasr/ggml/include/ggml-opencl.h +26 -0
- data/vendor/crispasr/ggml/include/ggml-openvino.h +37 -0
- data/vendor/crispasr/ggml/include/ggml-opt.h +256 -0
- data/vendor/crispasr/ggml/include/ggml-rpc.h +35 -0
- data/vendor/crispasr/ggml/include/ggml-sycl.h +49 -0
- data/vendor/crispasr/ggml/include/ggml-virtgpu.h +14 -0
- data/vendor/crispasr/ggml/include/ggml-vulkan.h +29 -0
- data/vendor/crispasr/ggml/include/ggml-webgpu.h +19 -0
- data/vendor/crispasr/ggml/include/ggml-zdnn.h +17 -0
- data/vendor/crispasr/ggml/include/ggml-zendnn.h +22 -0
- data/vendor/crispasr/ggml/include/ggml.h +2887 -0
- data/vendor/crispasr/ggml/include/gguf.h +204 -0
- data/vendor/crispasr/ggml/src/CMakeLists.txt +493 -0
- data/vendor/crispasr/ggml/src/ggml-alloc.c +1323 -0
- data/vendor/crispasr/ggml/src/ggml-backend-dl.cpp +48 -0
- data/vendor/crispasr/ggml/src/ggml-backend-dl.h +44 -0
- data/vendor/crispasr/ggml/src/ggml-backend-impl.h +275 -0
- data/vendor/crispasr/ggml/src/ggml-backend-meta.cpp +2145 -0
- data/vendor/crispasr/ggml/src/ggml-backend-reg.cpp +586 -0
- data/vendor/crispasr/ggml/src/ggml-backend.cpp +2437 -0
- data/vendor/crispasr/ggml/src/ggml-common.h +1900 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/CMakeLists.txt +718 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/amx/amx.h +8 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/amx/common.h +115 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/amx/mmq.cpp +2512 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/amx/mmq.h +10 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +98 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/arm/quants.c +4244 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/loongarch/quants.c +2158 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/riscv/quants.c +4455 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/wasm/quants.c +1220 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/x86/quants.c +3970 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch-fallback.h +349 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/binary-ops.h +16 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/common.h +95 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/ggml-cpu.c +3791 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/ggml-cpu.cpp +703 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/hbm.cpp +55 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/hbm.h +8 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kernels.cpp +939 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kernels.h +90 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1513 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4051 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/ops.cpp +11662 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/ops.h +121 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/quants.c +1288 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/quants.h +103 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/repack.cpp +4836 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/repack.h +245 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/simd-gemm.h +226 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/simd-mappings.h +1329 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime.cpp +1025 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime.h +13 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +3196 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime_kernels.h +26 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/traits.cpp +36 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/traits.h +38 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/unary-ops.cpp +336 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/unary-ops.h +35 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/vec.cpp +681 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/vec.h +1606 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/CMakeLists.txt +272 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/acc.cu +61 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/acc.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/add-id.cu +58 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/add-id.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/arange.cu +34 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/arange.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/argmax.cu +91 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/argmax.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/argsort.cu +265 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/argsort.cuh +19 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/binbcast.cu +534 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/binbcast.cuh +12 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/clamp.cu +45 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/clamp.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/common.cuh +1489 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/concat.cu +204 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/concat.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/conv-transpose-1d.cu +97 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/conv2d.cu +166 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/conv2d.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/convert.cu +892 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/convert.cuh +66 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/count-equal.cu +64 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/cpy.cu +581 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/cpy.cuh +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/cumsum.cu +307 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/cumsum.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/dequantize.cuh +99 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/diag.cu +77 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/diag.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/diagmask.cu +40 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn-common.cuh +1212 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1860 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn-tile.cu +57 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn-tile.cuh +1309 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn-vec.cuh +600 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn-wmma-f16.cu +696 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +51 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn.cu +620 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fill.cu +37 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fill.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/gated_delta_net.cu +273 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/gated_delta_net.cuh +4 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/getrows.cu +332 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/getrows.cuh +15 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/ggml-cuda.cu +5580 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/gla.cu +93 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/gla.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/im2col.cu +274 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/im2col.cuh +6 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mean.cu +75 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mean.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mma.cuh +1333 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmf.cu +191 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmf.cuh +908 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmid.cu +164 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmid.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmq.cu +372 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmq.cuh +4175 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmvf.cu +862 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmvf.cuh +14 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmvq.cu +1161 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmvq.cuh +16 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/norm.cu +756 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/norm.cuh +20 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/out-prod.cu +68 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/pad.cu +106 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/pad.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/pool2d.cu +94 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/quantize.cu +443 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/quantize.cuh +41 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/reduce_rows.cuh +39 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/roll.cu +67 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/roll.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/rope.cu +665 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/rope.cuh +9 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/scale.cu +34 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/scale.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/set-rows.cu +330 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/set-rows.cuh +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/set.cu +39 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/set.cuh +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/softcap.cu +34 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/softcap.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/softmax.cu +472 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/softmax.cuh +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/solve_tri.cu +275 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/ssm-conv.cu +197 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/ssm-scan.cu +342 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/sum.cu +41 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/sum.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/sumrows.cu +43 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/sumrows.cuh +4 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +11 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +11 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +11 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +11 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/top-k.cu +95 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/top-k.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/topk-moe.cu +415 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/topk-moe.cuh +27 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/tri.cu +136 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/tri.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/tsembd.cu +47 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/unary.cu +644 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/unary.cuh +116 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/upscale.cu +293 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/upscale.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/vecdotq.cuh +1317 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/vendors/hip.h +306 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/vendors/musa.h +148 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/wkv.cu +199 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/wkv.cuh +7 -0
- data/vendor/crispasr/ggml/src/ggml-impl.h +783 -0
- data/vendor/crispasr/ggml/src/ggml-metal/CMakeLists.txt +124 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-common.cpp +457 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-common.h +52 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-context.m +778 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-device.cpp +2117 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-device.h +299 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-device.m +2093 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-impl.h +1267 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-ops.cpp +5023 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-ops.h +111 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal.cpp +954 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal.metal +11756 -0
- data/vendor/crispasr/ggml/src/ggml-opt.cpp +1094 -0
- data/vendor/crispasr/ggml/src/ggml-quants.c +5491 -0
- data/vendor/crispasr/ggml/src/ggml-quants.h +112 -0
- data/vendor/crispasr/ggml/src/ggml-threading.cpp +12 -0
- data/vendor/crispasr/ggml/src/ggml-threading.h +14 -0
- data/vendor/crispasr/ggml/src/ggml.c +7925 -0
- data/vendor/crispasr/ggml/src/ggml.cpp +26 -0
- data/vendor/crispasr/ggml/src/gguf.cpp +1556 -0
- data/vendor/crispasr/src/cohere-arch.h +137 -0
- data/vendor/crispasr/src/cohere.cpp +5642 -0
- data/vendor/crispasr/src/cohere.h +327 -0
- data/vendor/crispasr/src/cohere_batch_planner.h +82 -0
- data/vendor/crispasr/src/cohere_chunking.h +64 -0
- data/vendor/crispasr/src/cohere_decoder_batch_layout.h +60 -0
- data/vendor/crispasr/src/cohere_encoder_padded_layout.h +27 -0
- data/vendor/crispasr/src/cohere_frontend.cpp +189 -0
- data/vendor/crispasr/src/cohere_frontend.h +31 -0
- data/vendor/crispasr/src/cohere_ragged_controller.h +149 -0
- data/vendor/crispasr/src/cohere_token_renderer.h +181 -0
- data/vendor/crispasr/src/core/attention.h +924 -0
- data/vendor/crispasr/src/core/audio_chunking.h +97 -0
- data/vendor/crispasr/src/core/beam_decode.h +486 -0
- data/vendor/crispasr/src/core/cpu_ops.h +135 -0
- data/vendor/crispasr/src/core/gguf_loader.cpp +1021 -0
- data/vendor/crispasr/src/core/gguf_loader.h +216 -0
- data/vendor/crispasr/src/core/gpu_backend_pref.h +119 -0
- data/vendor/crispasr/src/core/mel.cpp +519 -0
- data/vendor/crispasr/src/core/mel.h +265 -0
- data/vendor/crispasr/src/core/ngram_loop_fix.h +173 -0
- data/vendor/crispasr/src/core/repetition_loop_guard.h +54 -0
- data/vendor/crispasr/src/crispasr_imatrix.cpp +255 -0
- data/vendor/crispasr/src/crispasr_imatrix.h +38 -0
- metadata +596 -0
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
// src/core/audio_chunking.h — boundary detection for long-form audio.
|
|
2
|
+
//
|
|
3
|
+
// Backends with bounded encoder windows (Cohere model rows are <= 35 s, parakeet/canary
|
|
4
|
+
// can also benefit) currently cut at exactly N * sample_rate samples,
|
|
5
|
+
// which slices mid-word and corrupts the transcript at chunk seams.
|
|
6
|
+
//
|
|
7
|
+
// `find_energy_min_split` scans a 1-D mono PCM segment in fixed-size
|
|
8
|
+
// non-overlapping windows and returns the start index of the lowest-RMS
|
|
9
|
+
// window — i.e. the quietest 100 ms within a search range. Cutting
|
|
10
|
+
// there avoids splitting a syllable in two and keeps the encoder /
|
|
11
|
+
// decoder operating on coherent acoustic boundaries.
|
|
12
|
+
//
|
|
13
|
+
// `split_at_energy_minima` wraps that into a chunker that yields
|
|
14
|
+
// [begin, end) sample ranges of length <= max_chunk_samples, choosing
|
|
15
|
+
// each cut from the last `boundary_context_samples` of the running
|
|
16
|
+
// window so we land in a quiet point near the cap. Mirrors the
|
|
17
|
+
// `_find_split_point_energy` / `split_audio_chunks_energy` helpers in
|
|
18
|
+
// nano-cohere-transcribe (Apache 2.0); ported to C++ for use across
|
|
19
|
+
// CrispASR's encoder backends.
|
|
20
|
+
#pragma once
|
|
21
|
+
|
|
22
|
+
#include <cmath>
|
|
23
|
+
#include <cstddef>
|
|
24
|
+
#include <utility>
|
|
25
|
+
#include <vector>
|
|
26
|
+
|
|
27
|
+
namespace audio_chunking {
|
|
28
|
+
|
|
29
|
+
// Return the start index (relative to `samples[search_start]`'s base
|
|
30
|
+
// position in the caller's frame) of the lowest-RMS non-overlapping
|
|
31
|
+
// window of length `win_samples` inside [search_start, search_end).
|
|
32
|
+
//
|
|
33
|
+
// `search_end - search_start <= win_samples` is treated as a degenerate
|
|
34
|
+
// case and returns the midpoint of the search range.
|
|
35
|
+
inline size_t find_energy_min_split(const float* samples, size_t search_start, size_t search_end, size_t win_samples) {
|
|
36
|
+
if (search_end <= search_start)
|
|
37
|
+
return search_start;
|
|
38
|
+
const size_t span = search_end - search_start;
|
|
39
|
+
if (span <= win_samples)
|
|
40
|
+
return search_start + span / 2;
|
|
41
|
+
double min_e = std::numeric_limits<double>::infinity();
|
|
42
|
+
size_t best = search_start;
|
|
43
|
+
// Match the Cohere reference's `range(0, span - win, win)`: the final
|
|
44
|
+
// window whose end is exactly search_end is deliberately not a candidate.
|
|
45
|
+
for (size_t i = 0; i + win_samples < span; i += win_samples) {
|
|
46
|
+
double s = 0.0;
|
|
47
|
+
for (size_t j = 0; j < win_samples; ++j) {
|
|
48
|
+
const float v = samples[search_start + i + j];
|
|
49
|
+
s += (double)v * (double)v;
|
|
50
|
+
}
|
|
51
|
+
const double e = std::sqrt(s / (double)win_samples);
|
|
52
|
+
if (e < min_e) {
|
|
53
|
+
min_e = e;
|
|
54
|
+
best = search_start + i;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return best;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Split `samples[0 .. n_samples)` into <= max_chunk_samples chunks,
|
|
61
|
+
// preferring cuts inside the last `search_window_samples` of each
|
|
62
|
+
// running window where RMS is lowest. Returns [begin, end) ranges in
|
|
63
|
+
// sample units. If the input is shorter than max_chunk_samples, returns
|
|
64
|
+
// a single [0, n_samples) range.
|
|
65
|
+
inline std::vector<std::pair<size_t, size_t>> split_at_energy_minima(const float* samples, size_t n_samples,
|
|
66
|
+
size_t max_chunk_samples,
|
|
67
|
+
size_t search_window_samples,
|
|
68
|
+
size_t win_samples = 1600) {
|
|
69
|
+
std::vector<std::pair<size_t, size_t>> out;
|
|
70
|
+
if (n_samples == 0)
|
|
71
|
+
return out;
|
|
72
|
+
if (max_chunk_samples == 0 || n_samples <= max_chunk_samples) {
|
|
73
|
+
out.emplace_back(0, n_samples);
|
|
74
|
+
return out;
|
|
75
|
+
}
|
|
76
|
+
size_t idx = 0;
|
|
77
|
+
while (idx < n_samples) {
|
|
78
|
+
if (idx + max_chunk_samples >= n_samples) {
|
|
79
|
+
out.emplace_back(idx, n_samples);
|
|
80
|
+
break;
|
|
81
|
+
}
|
|
82
|
+
const size_t search_start =
|
|
83
|
+
(search_window_samples >= max_chunk_samples) ? idx : idx + max_chunk_samples - search_window_samples;
|
|
84
|
+
const size_t search_end = idx + max_chunk_samples;
|
|
85
|
+
size_t cut = find_energy_min_split(samples, search_start, search_end, win_samples);
|
|
86
|
+
// Defensive clamp: forward progress, never past the cap.
|
|
87
|
+
if (cut <= idx)
|
|
88
|
+
cut = idx + max_chunk_samples;
|
|
89
|
+
if (cut > n_samples)
|
|
90
|
+
cut = n_samples;
|
|
91
|
+
out.emplace_back(idx, cut);
|
|
92
|
+
idx = cut;
|
|
93
|
+
}
|
|
94
|
+
return out;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
} // namespace audio_chunking
|
|
@@ -0,0 +1,486 @@
|
|
|
1
|
+
// src/core/beam_decode.h — shared autoregressive beam-search decode loop.
|
|
2
|
+
//
|
|
3
|
+
// Companion to `core_greedy_decode::run_with_probs`. Same callback shape
|
|
4
|
+
// at the call site (a single `replay_fn`), but instead of picking one
|
|
5
|
+
// argmax / sampled token per step, this helper expands the top
|
|
6
|
+
// `beam_size` hypotheses in parallel and prunes globally to keep the
|
|
7
|
+
// highest-cumulative-logprob beams alive. Returns the winning beam's
|
|
8
|
+
// tokens + per-token softmax probabilities.
|
|
9
|
+
//
|
|
10
|
+
// KV strategy — replay-from-prefix
|
|
11
|
+
// --------------------------------
|
|
12
|
+
// Beam search needs each beam to have its own KV state, but the LLM-style
|
|
13
|
+
// runtime backends (glm-asr, kyutai-stt, omniasr-llm, moonshine via
|
|
14
|
+
// decode step) keep KV in a single context-owned buffer. To avoid adding
|
|
15
|
+
// a per-backend `kv_save` / `kv_restore` API, we exploit the fact that
|
|
16
|
+
// every call to "advance the LM by these tokens at this n_past" lets us
|
|
17
|
+
// write KV slots at any logical position. Each step rebuilds each beam's
|
|
18
|
+
// KV by replaying its full generated suffix from the post-prompt anchor.
|
|
19
|
+
//
|
|
20
|
+
// Cost: O(beam_size × T²/2) extra forward work for T generated tokens vs
|
|
21
|
+
// greedy's O(T). Acceptable for beam_size = 2-4 since the audio encoder
|
|
22
|
+
// typically dominates wall time on these backends. If perf is bad on
|
|
23
|
+
// long generations, the right next step is `*_kv_save` / `*_kv_restore`
|
|
24
|
+
// per backend, not making this helper smarter.
|
|
25
|
+
//
|
|
26
|
+
// Caller contract
|
|
27
|
+
// ---------------
|
|
28
|
+
// Caller is responsible for:
|
|
29
|
+
// * Running the prompt prefill so KV slots [0, prompt_len) hold the
|
|
30
|
+
// prompt's K/V.
|
|
31
|
+
// * Capturing the prefill logits at the last prompt position.
|
|
32
|
+
// * Providing a `replay_fn(ctx, tokens, n_tokens, prompt_len)` that
|
|
33
|
+
// overwrites KV slots [prompt_len, prompt_len + n_tokens) with the
|
|
34
|
+
// given suffix's K/V and returns the last-position logits as a
|
|
35
|
+
// malloc'd `float*` of size `vocab_size` (or nullptr on failure).
|
|
36
|
+
// The helper free()s the returned buffer.
|
|
37
|
+
// * Resetting KV after `run_with_probs` returns (state is undefined
|
|
38
|
+
// since each beam-step overwrites slots).
|
|
39
|
+
//
|
|
40
|
+
// For backends that natively expose a batched
|
|
41
|
+
// `forward(ctx, embeds, n_tokens, n_past)` — like glm-asr's
|
|
42
|
+
// `glm_asr_run_llm_kv` — the replay_fn is a one-liner that embeds +
|
|
43
|
+
// forwards. For backends with a per-token API (like omniasr-llm's
|
|
44
|
+
// `omniasr_run_dec_token`), the replay_fn loops over tokens internally.
|
|
45
|
+
//
|
|
46
|
+
// Usage (glm-asr):
|
|
47
|
+
//
|
|
48
|
+
// core_beam_decode::Config cfg;
|
|
49
|
+
// cfg.max_new_tokens = 512;
|
|
50
|
+
// cfg.eos_id = hp.eos_token_ids[0];
|
|
51
|
+
// cfg.vocab_size = hp.llm_vocab;
|
|
52
|
+
// cfg.beam_size = params.beam_size;
|
|
53
|
+
// cfg.prompt_len = (int)prompt_ids.size();
|
|
54
|
+
//
|
|
55
|
+
// auto replay = [ctx](const int32_t* toks, int n, int prompt_len) -> float* {
|
|
56
|
+
// float* emb = glm_asr_embed_tokens(ctx, toks, n);
|
|
57
|
+
// if (!emb) return nullptr;
|
|
58
|
+
// float* lg = glm_asr_run_llm_kv(ctx, emb, n, prompt_len, nullptr, nullptr);
|
|
59
|
+
// std::free(emb);
|
|
60
|
+
// return lg;
|
|
61
|
+
// };
|
|
62
|
+
//
|
|
63
|
+
// auto r = core_beam_decode::run_with_probs(ctx, prefill_lg, replay, cfg);
|
|
64
|
+
//
|
|
65
|
+
// Header-only so the compiler inlines each caller's concrete callable
|
|
66
|
+
// at the call site (matching the greedy helper's pattern).
|
|
67
|
+
|
|
68
|
+
#pragma once
|
|
69
|
+
|
|
70
|
+
#include <algorithm>
|
|
71
|
+
#include <cmath>
|
|
72
|
+
#include <cstdint>
|
|
73
|
+
#include <cstdlib>
|
|
74
|
+
#include <memory>
|
|
75
|
+
#include <type_traits>
|
|
76
|
+
#include <utility>
|
|
77
|
+
#include <vector>
|
|
78
|
+
|
|
79
|
+
namespace core_beam_decode {
|
|
80
|
+
|
|
81
|
+
struct Config {
|
|
82
|
+
int max_new_tokens = 512; // hard cap on generated tokens
|
|
83
|
+
int eos_id = 2; // legacy single-EOS field; used iff eos_ids is empty
|
|
84
|
+
std::vector<int> eos_ids; // multi-EOS set (e.g. glm-asr's 3 stop tokens). Any match terminates the beam.
|
|
85
|
+
int vocab_size = 0; // required
|
|
86
|
+
int beam_size = 1; // 1 = degenerate to greedy-via-beam (still works, just expensive)
|
|
87
|
+
int prompt_len = 0; // n_past after prompt prefill (replay anchor)
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
struct Result {
|
|
91
|
+
std::vector<int32_t> tokens; // generated tokens of the winning beam
|
|
92
|
+
std::vector<float> probs; // softmax prob of each picked token in [0,1]
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
namespace detail {
|
|
96
|
+
|
|
97
|
+
// Internal beam state. Tokens are the generated suffix only (post-prompt).
|
|
98
|
+
struct Beam {
|
|
99
|
+
std::vector<int32_t> tokens;
|
|
100
|
+
std::vector<float> probs;
|
|
101
|
+
double cum_logprob = 0.0;
|
|
102
|
+
bool finished = false;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
// Numerically-stable log-Z (log normaliser of softmax over `logits`).
|
|
106
|
+
inline double compute_logZ(const float* logits, int vocab) {
|
|
107
|
+
float mx = logits[0];
|
|
108
|
+
for (int k = 1; k < vocab; k++)
|
|
109
|
+
if (logits[k] > mx)
|
|
110
|
+
mx = logits[k];
|
|
111
|
+
double sum = 0.0;
|
|
112
|
+
for (int k = 0; k < vocab; k++)
|
|
113
|
+
sum += std::exp((double)(logits[k] - mx));
|
|
114
|
+
return (double)mx + std::log(sum);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Pick top-K tokens from `logits` and return their log-softmax values.
|
|
118
|
+
// `out_ids` and `out_lps` are sized to K, sorted descending by logit.
|
|
119
|
+
//
|
|
120
|
+
// Uses a min-heap of size K: O(V log K) instead of the previous
|
|
121
|
+
// O(V + V log K) partial_sort that allocated a full vocab-sized index
|
|
122
|
+
// vector per call (§176r). For beam_size=4 and vocab=32K-150K this
|
|
123
|
+
// eliminates a ~128-600 KB alloc per beam expansion step.
|
|
124
|
+
inline void top_k_log_softmax(const float* logits, int vocab, int K, std::vector<int>& out_ids,
|
|
125
|
+
std::vector<double>& out_lps) {
|
|
126
|
+
if (K > vocab)
|
|
127
|
+
K = vocab;
|
|
128
|
+
out_ids.resize((size_t)K);
|
|
129
|
+
out_lps.resize((size_t)K);
|
|
130
|
+
|
|
131
|
+
const double logZ = compute_logZ(logits, vocab);
|
|
132
|
+
|
|
133
|
+
// Min-heap: stores (logit, token_id) pairs; the smallest logit is
|
|
134
|
+
// at the top so we can cheaply evict it when a larger one arrives.
|
|
135
|
+
// Reused across calls (thread-local) so the K-element heap isn't
|
|
136
|
+
// re-allocated on every beam-expansion step (B×T per transcribe). clear()
|
|
137
|
+
// keeps capacity; the heap is rebuilt from scratch below, so the result is
|
|
138
|
+
// bit-identical to a fresh vector. (§176r follow-up)
|
|
139
|
+
using Pair = std::pair<float, int>;
|
|
140
|
+
static thread_local std::vector<Pair> heap;
|
|
141
|
+
heap.clear();
|
|
142
|
+
heap.reserve((size_t)K);
|
|
143
|
+
for (int i = 0; i < vocab; i++) {
|
|
144
|
+
if ((int)heap.size() < K) {
|
|
145
|
+
heap.push_back({logits[i], i});
|
|
146
|
+
if ((int)heap.size() == K)
|
|
147
|
+
std::make_heap(heap.begin(), heap.end(),
|
|
148
|
+
[](const Pair& a, const Pair& b) { return a.first > b.first; });
|
|
149
|
+
} else if (logits[i] > heap[0].first) {
|
|
150
|
+
std::pop_heap(heap.begin(), heap.end(), [](const Pair& a, const Pair& b) { return a.first > b.first; });
|
|
151
|
+
heap.back() = {logits[i], i};
|
|
152
|
+
std::push_heap(heap.begin(), heap.end(), [](const Pair& a, const Pair& b) { return a.first > b.first; });
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
// Sort descending by logit.
|
|
156
|
+
std::sort(heap.begin(), heap.end(), [](const Pair& a, const Pair& b) { return a.first > b.first; });
|
|
157
|
+
for (int i = 0; i < K; i++) {
|
|
158
|
+
out_ids[(size_t)i] = heap[(size_t)i].second;
|
|
159
|
+
out_lps[(size_t)i] = (double)heap[(size_t)i].first - logZ;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
} // namespace detail
|
|
164
|
+
|
|
165
|
+
// Run the beam decode loop.
|
|
166
|
+
//
|
|
167
|
+
// Template parameters:
|
|
168
|
+
// Ctx : the model's opaque context type.
|
|
169
|
+
// ReplayFn : callable with signature
|
|
170
|
+
// `float * (Ctx*, const int32_t* tokens, int n_tokens, int prompt_len)`.
|
|
171
|
+
// Must overwrite KV slots [prompt_len, prompt_len + n_tokens)
|
|
172
|
+
// and return a malloc'd `float*` containing the last-position
|
|
173
|
+
// logits ([vocab_size, 1]). Returning nullptr terminates that
|
|
174
|
+
// beam.
|
|
175
|
+
//
|
|
176
|
+
// Pre-condition: caller has populated KV slots [0, prompt_len) with the
|
|
177
|
+
// prompt's K/V; `prefill_logits` is the [V, 1] logits at the last prompt
|
|
178
|
+
// position (caller-owned, not freed by this helper).
|
|
179
|
+
//
|
|
180
|
+
// Post-condition: KV is left in an undefined state — caller must reset
|
|
181
|
+
// it before the next transcription.
|
|
182
|
+
template <typename Ctx, typename ReplayFn>
|
|
183
|
+
inline Result run_with_probs(Ctx* ctx, const float* prefill_logits, ReplayFn replay_fn, const Config& cfg) {
|
|
184
|
+
using detail::Beam;
|
|
185
|
+
|
|
186
|
+
Result result;
|
|
187
|
+
if (!prefill_logits || cfg.vocab_size <= 0)
|
|
188
|
+
return result;
|
|
189
|
+
|
|
190
|
+
const int B = (cfg.beam_size > 0) ? cfg.beam_size : 1;
|
|
191
|
+
const int V = cfg.vocab_size;
|
|
192
|
+
|
|
193
|
+
auto is_eos = [&](int id) {
|
|
194
|
+
if (!cfg.eos_ids.empty()) {
|
|
195
|
+
for (int e : cfg.eos_ids)
|
|
196
|
+
if (id == e)
|
|
197
|
+
return true;
|
|
198
|
+
return false;
|
|
199
|
+
}
|
|
200
|
+
return id == cfg.eos_id;
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
// 1. Initial beams: top-B from prefill logits.
|
|
204
|
+
std::vector<int> first_ids;
|
|
205
|
+
std::vector<double> first_lps;
|
|
206
|
+
detail::top_k_log_softmax(prefill_logits, V, B, first_ids, first_lps);
|
|
207
|
+
|
|
208
|
+
std::vector<Beam> beams((size_t)first_ids.size());
|
|
209
|
+
for (size_t i = 0; i < first_ids.size(); i++) {
|
|
210
|
+
beams[i].tokens.push_back(first_ids[i]);
|
|
211
|
+
beams[i].probs.push_back((float)std::exp(first_lps[i]));
|
|
212
|
+
beams[i].cum_logprob = first_lps[i];
|
|
213
|
+
if (is_eos(first_ids[i]))
|
|
214
|
+
beams[i].finished = true;
|
|
215
|
+
}
|
|
216
|
+
// top_k_log_softmax already returns descending; beams are sorted.
|
|
217
|
+
|
|
218
|
+
if (beams.empty())
|
|
219
|
+
return result;
|
|
220
|
+
|
|
221
|
+
// 2. Per-step expand-and-prune.
|
|
222
|
+
while ((int)beams[0].tokens.size() < cfg.max_new_tokens && !beams[0].finished) {
|
|
223
|
+
struct Cand {
|
|
224
|
+
int beam_idx;
|
|
225
|
+
int token;
|
|
226
|
+
double cum_logprob;
|
|
227
|
+
float token_prob;
|
|
228
|
+
bool from_finished; // carry-forward of an already-finished beam
|
|
229
|
+
};
|
|
230
|
+
std::vector<Cand> cands;
|
|
231
|
+
cands.reserve((size_t)B * (size_t)B + (size_t)B);
|
|
232
|
+
|
|
233
|
+
for (size_t bi = 0; bi < beams.size(); bi++) {
|
|
234
|
+
auto& b = beams[bi];
|
|
235
|
+
if (b.finished) {
|
|
236
|
+
// Carry forward; the token field is only used by the
|
|
237
|
+
// detokeniser, which skips finished beams.
|
|
238
|
+
const int sentinel = cfg.eos_ids.empty() ? cfg.eos_id : cfg.eos_ids[0];
|
|
239
|
+
cands.push_back({(int)bi, sentinel, b.cum_logprob, 1.0f, true});
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
float* lg = replay_fn(ctx, b.tokens.data(), (int)b.tokens.size(), cfg.prompt_len);
|
|
244
|
+
if (!lg) {
|
|
245
|
+
b.finished = true;
|
|
246
|
+
continue;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
std::vector<int> ids;
|
|
250
|
+
std::vector<double> lps;
|
|
251
|
+
detail::top_k_log_softmax(lg, V, B, ids, lps);
|
|
252
|
+
std::free(lg);
|
|
253
|
+
|
|
254
|
+
for (size_t j = 0; j < ids.size(); j++) {
|
|
255
|
+
Cand c;
|
|
256
|
+
c.beam_idx = (int)bi;
|
|
257
|
+
c.token = ids[j];
|
|
258
|
+
c.cum_logprob = b.cum_logprob + lps[j];
|
|
259
|
+
c.token_prob = (float)std::exp(lps[j]);
|
|
260
|
+
c.from_finished = false;
|
|
261
|
+
cands.push_back(c);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
if (cands.empty())
|
|
266
|
+
break;
|
|
267
|
+
|
|
268
|
+
const size_t keep = std::min<size_t>((size_t)B, cands.size());
|
|
269
|
+
std::partial_sort(cands.begin(), cands.begin() + keep, cands.end(),
|
|
270
|
+
[](const Cand& a, const Cand& b) { return a.cum_logprob > b.cum_logprob; });
|
|
271
|
+
cands.resize(keep);
|
|
272
|
+
|
|
273
|
+
std::vector<Beam> next_beams;
|
|
274
|
+
next_beams.reserve(keep);
|
|
275
|
+
for (auto& c : cands) {
|
|
276
|
+
Beam nb = beams[(size_t)c.beam_idx]; // copy parent
|
|
277
|
+
if (c.from_finished) {
|
|
278
|
+
next_beams.push_back(std::move(nb));
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
nb.tokens.push_back(c.token);
|
|
282
|
+
nb.probs.push_back(c.token_prob);
|
|
283
|
+
nb.cum_logprob = c.cum_logprob;
|
|
284
|
+
if (is_eos(c.token))
|
|
285
|
+
nb.finished = true;
|
|
286
|
+
next_beams.push_back(std::move(nb));
|
|
287
|
+
}
|
|
288
|
+
beams = std::move(next_beams);
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// 3. Best beam is beams[0] (top-of-cands ordering preserved by partial_sort).
|
|
292
|
+
result.tokens = std::move(beams[0].tokens);
|
|
293
|
+
result.probs = std::move(beams[0].probs);
|
|
294
|
+
return result;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
// ---------------------------------------------------------------------------
|
|
298
|
+
// Branched variant: per-beam KV snapshots (O(B × T) single-token forwards).
|
|
299
|
+
// ---------------------------------------------------------------------------
|
|
300
|
+
//
|
|
301
|
+
// Use this when the backend's per-step decode is one-token-at-a-time and the
|
|
302
|
+
// caller can cheaply snapshot/restore the model's KV state. Gives true
|
|
303
|
+
// O(B × T) single-token forwards instead of replay-from-prefix's O(B × T²).
|
|
304
|
+
//
|
|
305
|
+
// Caller provides four callbacks plus the post-prefill prompt logits:
|
|
306
|
+
// save_fn(ctx) -> Snap (snapshot of current KV state)
|
|
307
|
+
// restore_fn(ctx, snap) -> void (writes snap back into ctx KV; snap
|
|
308
|
+
// remains valid for further restores)
|
|
309
|
+
// snap_free_fn(snap) -> void (called once when the snap is dead)
|
|
310
|
+
// step_fn(ctx, tok, n_past)-> float* (writes KV slot at n_past for `tok`,
|
|
311
|
+
// returns malloc'd [vocab_size] logits;
|
|
312
|
+
// helper free()s it. nullptr on failure.)
|
|
313
|
+
//
|
|
314
|
+
// The Snap type is whatever save_fn returns — typically a heap pointer to a
|
|
315
|
+
// caller-defined struct. The helper wraps it in a refcounted holder so
|
|
316
|
+
// siblings can share a parent's snapshot without double-free. Restore is
|
|
317
|
+
// read-only on the snap (it copies snap into ctx); only step_fn / save_fn
|
|
318
|
+
// mutate ctx KV.
|
|
319
|
+
//
|
|
320
|
+
// Pre-condition: KV slots [0, prompt_len) hold the prompt's K/V; the helper
|
|
321
|
+
// snapshots that state once at entry. `prefill_logits` is the [V, 1] logits
|
|
322
|
+
// at the last prompt position (caller-owned, not freed).
|
|
323
|
+
template <typename Ctx, typename SaveFn, typename RestoreFn, typename SnapFreeFn, typename StepFn>
|
|
324
|
+
inline Result run_with_probs_branched(Ctx* ctx, const float* prefill_logits, SaveFn save_fn, RestoreFn restore_fn,
|
|
325
|
+
SnapFreeFn snap_free_fn, StepFn step_fn, const Config& cfg) {
|
|
326
|
+
using Snap = decltype(save_fn(ctx));
|
|
327
|
+
|
|
328
|
+
Result result;
|
|
329
|
+
if (!prefill_logits || cfg.vocab_size <= 0)
|
|
330
|
+
return result;
|
|
331
|
+
|
|
332
|
+
const int B = (cfg.beam_size > 0) ? cfg.beam_size : 1;
|
|
333
|
+
const int V = cfg.vocab_size;
|
|
334
|
+
|
|
335
|
+
auto is_eos = [&](int id) {
|
|
336
|
+
if (!cfg.eos_ids.empty()) {
|
|
337
|
+
for (int e : cfg.eos_ids)
|
|
338
|
+
if (id == e)
|
|
339
|
+
return true;
|
|
340
|
+
return false;
|
|
341
|
+
}
|
|
342
|
+
return id == cfg.eos_id;
|
|
343
|
+
};
|
|
344
|
+
|
|
345
|
+
// RAII wrapper so siblings can share a parent's snap by shared_ptr.
|
|
346
|
+
// The destructor calls snap_free_fn exactly once when the last ref dies.
|
|
347
|
+
struct Holder {
|
|
348
|
+
Snap snap;
|
|
349
|
+
SnapFreeFn* free_cb;
|
|
350
|
+
Holder(Snap s, SnapFreeFn* f) : snap(s), free_cb(f) {}
|
|
351
|
+
~Holder() { (*free_cb)(snap); }
|
|
352
|
+
Holder(const Holder&) = delete;
|
|
353
|
+
Holder& operator=(const Holder&) = delete;
|
|
354
|
+
};
|
|
355
|
+
auto wrap = [&snap_free_fn](Snap s) { return std::shared_ptr<Holder>(new Holder(s, &snap_free_fn)); };
|
|
356
|
+
|
|
357
|
+
struct BeamS {
|
|
358
|
+
std::vector<int32_t> tokens;
|
|
359
|
+
std::vector<float> probs;
|
|
360
|
+
double cum_logprob = 0.0;
|
|
361
|
+
bool finished = false;
|
|
362
|
+
// KV state right BEFORE feeding tokens.back() to step_fn. All initial
|
|
363
|
+
// beams share the prompt snap; siblings share their parent's
|
|
364
|
+
// post-step snap. Restored at the start of each per-beam expand.
|
|
365
|
+
std::shared_ptr<Holder> snap;
|
|
366
|
+
};
|
|
367
|
+
|
|
368
|
+
// 1. Snapshot the post-prefill prompt KV; seed initial beams from
|
|
369
|
+
// top-K of prefill_logits. No step_fn calls happen during seeding —
|
|
370
|
+
// the first round of the loop below will do that for each beam.
|
|
371
|
+
auto prompt_snap = wrap(save_fn(ctx));
|
|
372
|
+
|
|
373
|
+
std::vector<int> first_ids;
|
|
374
|
+
std::vector<double> first_lps;
|
|
375
|
+
detail::top_k_log_softmax(prefill_logits, V, B, first_ids, first_lps);
|
|
376
|
+
|
|
377
|
+
std::vector<BeamS> beams((size_t)first_ids.size());
|
|
378
|
+
for (size_t i = 0; i < first_ids.size(); i++) {
|
|
379
|
+
beams[i].tokens.push_back(first_ids[i]);
|
|
380
|
+
beams[i].probs.push_back((float)std::exp(first_lps[i]));
|
|
381
|
+
beams[i].cum_logprob = first_lps[i];
|
|
382
|
+
if (is_eos(first_ids[i]))
|
|
383
|
+
beams[i].finished = true;
|
|
384
|
+
beams[i].snap = prompt_snap;
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
if (beams.empty())
|
|
388
|
+
return result;
|
|
389
|
+
|
|
390
|
+
// 2. Per-round expand + prune.
|
|
391
|
+
//
|
|
392
|
+
// For each unfinished beam: restore beam.snap (which is the KV state
|
|
393
|
+
// RIGHT BEFORE feeding beam.tokens.back()), call step_fn for that token
|
|
394
|
+
// — which writes its KV slot and returns logits for the NEXT token —
|
|
395
|
+
// then capture a fresh snap (state right AFTER that token). Take top-K
|
|
396
|
+
// from the logits as expansion candidates; siblings inherit the same
|
|
397
|
+
// post-step snap shared_ptr.
|
|
398
|
+
while ((int)beams[0].tokens.size() < cfg.max_new_tokens && !beams[0].finished) {
|
|
399
|
+
struct Cand {
|
|
400
|
+
int beam_idx;
|
|
401
|
+
int token;
|
|
402
|
+
double cum_logprob;
|
|
403
|
+
float token_prob;
|
|
404
|
+
bool from_finished;
|
|
405
|
+
};
|
|
406
|
+
std::vector<Cand> cands;
|
|
407
|
+
cands.reserve((size_t)B * (size_t)B + (size_t)B);
|
|
408
|
+
|
|
409
|
+
// Per-beam post-step snap — shared by all surviving children.
|
|
410
|
+
std::vector<std::shared_ptr<Holder>> step_snaps(beams.size());
|
|
411
|
+
|
|
412
|
+
for (size_t bi = 0; bi < beams.size(); bi++) {
|
|
413
|
+
auto& b = beams[bi];
|
|
414
|
+
if (b.finished) {
|
|
415
|
+
const int sentinel = cfg.eos_ids.empty() ? cfg.eos_id : cfg.eos_ids[0];
|
|
416
|
+
cands.push_back({(int)bi, sentinel, b.cum_logprob, 1.0f, true});
|
|
417
|
+
continue;
|
|
418
|
+
}
|
|
419
|
+
// n_past = number of tokens preceding the one we're feeding.
|
|
420
|
+
// Beam currently holds (b.tokens.size()) chosen tokens; the
|
|
421
|
+
// first (b.tokens.size() - 1) of them have already been stepped
|
|
422
|
+
// through the LM in prior rounds. Now we step the most recent
|
|
423
|
+
// one at slot prompt_len + (b.tokens.size() - 1).
|
|
424
|
+
const int n_past = cfg.prompt_len + (int)b.tokens.size() - 1;
|
|
425
|
+
restore_fn(ctx, b.snap->snap);
|
|
426
|
+
float* lg = step_fn(ctx, b.tokens.back(), n_past);
|
|
427
|
+
if (!lg) {
|
|
428
|
+
b.finished = true;
|
|
429
|
+
continue;
|
|
430
|
+
}
|
|
431
|
+
step_snaps[bi] = wrap(save_fn(ctx));
|
|
432
|
+
|
|
433
|
+
std::vector<int> ids;
|
|
434
|
+
std::vector<double> lps;
|
|
435
|
+
detail::top_k_log_softmax(lg, V, B, ids, lps);
|
|
436
|
+
std::free(lg);
|
|
437
|
+
|
|
438
|
+
for (size_t j = 0; j < ids.size(); j++) {
|
|
439
|
+
cands.push_back({(int)bi, ids[j], b.cum_logprob + lps[j], (float)std::exp(lps[j]), false});
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
if (cands.empty())
|
|
444
|
+
break;
|
|
445
|
+
|
|
446
|
+
const size_t keep = std::min<size_t>((size_t)B, cands.size());
|
|
447
|
+
std::partial_sort(cands.begin(), cands.begin() + keep, cands.end(),
|
|
448
|
+
[](const Cand& a, const Cand& b) { return a.cum_logprob > b.cum_logprob; });
|
|
449
|
+
cands.resize(keep);
|
|
450
|
+
|
|
451
|
+
std::vector<BeamS> next_beams;
|
|
452
|
+
next_beams.reserve(keep);
|
|
453
|
+
for (auto& c : cands) {
|
|
454
|
+
const auto& parent = beams[(size_t)c.beam_idx];
|
|
455
|
+
BeamS nb;
|
|
456
|
+
nb.tokens = parent.tokens;
|
|
457
|
+
nb.probs = parent.probs;
|
|
458
|
+
nb.cum_logprob = parent.cum_logprob;
|
|
459
|
+
nb.finished = parent.finished;
|
|
460
|
+
if (c.from_finished) {
|
|
461
|
+
// Carry forward unchanged; same snap stays valid since no
|
|
462
|
+
// step_fn ran for this beam this round.
|
|
463
|
+
nb.snap = parent.snap;
|
|
464
|
+
next_beams.push_back(std::move(nb));
|
|
465
|
+
continue;
|
|
466
|
+
}
|
|
467
|
+
nb.tokens.push_back(c.token);
|
|
468
|
+
nb.probs.push_back(c.token_prob);
|
|
469
|
+
nb.cum_logprob = c.cum_logprob;
|
|
470
|
+
if (is_eos(c.token))
|
|
471
|
+
nb.finished = true;
|
|
472
|
+
nb.snap = step_snaps[(size_t)c.beam_idx];
|
|
473
|
+
next_beams.push_back(std::move(nb));
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
beams = std::move(next_beams);
|
|
477
|
+
// Old `beams` shared_ptrs and unselected `step_snaps` go out of scope
|
|
478
|
+
// here — Holder destructors fire snap_free_fn exactly once each.
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
result.tokens = std::move(beams[0].tokens);
|
|
482
|
+
result.probs = std::move(beams[0].probs);
|
|
483
|
+
return result;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
} // namespace core_beam_decode
|