cohere-transcribe 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +21 -0
- data/LICENSE.txt +201 -0
- data/NOTICE +5 -0
- data/README.md +265 -0
- data/THIRD_PARTY_NOTICES.md +67 -0
- data/exe/cohere-transcribe +8 -0
- data/exe/cohere-transcribe-doctor +8 -0
- data/ext/cohere_transcribe_native/CMakeLists.txt +262 -0
- data/ext/cohere_transcribe_native/README.md +26 -0
- data/ext/cohere_transcribe_native/audio_abi.cpp +1416 -0
- data/ext/cohere_transcribe_native/audio_exports.macos +5 -0
- data/ext/cohere_transcribe_native/audio_exports.map +10 -0
- data/ext/cohere_transcribe_native/cohere_abi.cpp +1209 -0
- data/ext/cohere_transcribe_native/exports.macos +40 -0
- data/ext/cohere_transcribe_native/exports.map +45 -0
- data/ext/cohere_transcribe_native/extconf.rb +89 -0
- data/ext/cohere_transcribe_native/test/abi_smoke.rb +232 -0
- data/ext/cohere_transcribe_native/test/audio_matrix_smoke.cpp +121 -0
- data/ext/cohere_transcribe_native/test/audio_reliability_smoke.cpp +198 -0
- data/ext/cohere_transcribe_native/test/native_abi_reliability_smoke.cpp +186 -0
- data/ext/cohere_transcribe_native/test/native_batch_projection_probe.rb +81 -0
- data/ext/cohere_transcribe_native/test/native_cancellation_smoke.cpp +194 -0
- data/lib/cohere/transcribe/alignment/ATTRIBUTION.md +23 -0
- data/lib/cohere/transcribe/alignment/LICENSE.ctc-forced-aligner +407 -0
- data/lib/cohere/transcribe/alignment/LICENSE.torchaudio +25 -0
- data/lib/cohere/transcribe/alignment/LICENSE.uroman +26 -0
- data/lib/cohere/transcribe/alignment/aligner.rb +476 -0
- data/lib/cohere/transcribe/alignment/ctc.rb +224 -0
- data/lib/cohere/transcribe/alignment/text.rb +237 -0
- data/lib/cohere/transcribe/alignment/uroman_data.rb +4950 -0
- data/lib/cohere/transcribe/api.rb +173 -0
- data/lib/cohere/transcribe/asr/batching.rb +472 -0
- data/lib/cohere/transcribe/asr/failure_policy.rb +112 -0
- data/lib/cohere/transcribe/asr/native.rb +676 -0
- data/lib/cohere/transcribe/audio/ATTRIBUTION.md +8 -0
- data/lib/cohere/transcribe/audio/LICENSE.auditok +21 -0
- data/lib/cohere/transcribe/audio/decoder.rb +315 -0
- data/lib/cohere/transcribe/audio/ffmpeg_native.rb +248 -0
- data/lib/cohere/transcribe/audio/segmentation.rb +260 -0
- data/lib/cohere/transcribe/cli.rb +727 -0
- data/lib/cohere/transcribe/configuration.rb +282 -0
- data/lib/cohere/transcribe/constants.rb +14 -0
- data/lib/cohere/transcribe/dense_converter.rb +548 -0
- data/lib/cohere/transcribe/doctor.rb +576 -0
- data/lib/cohere/transcribe/errors.rb +57 -0
- data/lib/cohere/transcribe/gguf_writer.rb +268 -0
- data/lib/cohere/transcribe/hub.rb +436 -0
- data/lib/cohere/transcribe/input.rb +110 -0
- data/lib/cohere/transcribe/licenses/crispasr.txt +21 -0
- data/lib/cohere/transcribe/loader.rb +128 -0
- data/lib/cohere/transcribe/model_identity.rb +440 -0
- data/lib/cohere/transcribe/output/publication.rb +1118 -0
- data/lib/cohere/transcribe/output/rendering.rb +105 -0
- data/lib/cohere/transcribe/output/timing.rb +86 -0
- data/lib/cohere/transcribe/python_text.rb +70 -0
- data/lib/cohere/transcribe/pytorch_checkpoint.rb +1180 -0
- data/lib/cohere/transcribe/runtime/engine.rb +1676 -0
- data/lib/cohere/transcribe/runtime/model_provider.rb +390 -0
- data/lib/cohere/transcribe/runtime/precision.rb +57 -0
- data/lib/cohere/transcribe/runtime/preparation.rb +215 -0
- data/lib/cohere/transcribe/runtime/resources.rb +165 -0
- data/lib/cohere/transcribe/runtime/word_pipeline.rb +364 -0
- data/lib/cohere/transcribe/safetensors.rb +579 -0
- data/lib/cohere/transcribe/state/checkpoint.rb +224 -0
- data/lib/cohere/transcribe/state/contracts.rb +141 -0
- data/lib/cohere/transcribe/state/io.rb +727 -0
- data/lib/cohere/transcribe/state/locking.rb +211 -0
- data/lib/cohere/transcribe/state/manifest.rb +155 -0
- data/lib/cohere/transcribe/state.rb +7 -0
- data/lib/cohere/transcribe/types.rb +535 -0
- data/lib/cohere/transcribe/vad/ATTRIBUTION.md +14 -0
- data/lib/cohere/transcribe/vad/LICENSE.faster-whisper +21 -0
- data/lib/cohere/transcribe/vad/LICENSE.silero-vad +21 -0
- data/lib/cohere/transcribe/vad/silero.rb +344 -0
- data/lib/cohere/transcribe/vad/silero_vad_v6.onnx +0 -0
- data/lib/cohere/transcribe/vad/timestamps.rb +219 -0
- data/lib/cohere/transcribe/version.rb +7 -0
- data/lib/cohere/transcribe.rb +26 -0
- data/sig/cohere/transcribe.rbs +250 -0
- data/vendor/crispasr/AUTHORS +510 -0
- data/vendor/crispasr/LICENSE +21 -0
- data/vendor/crispasr/UPSTREAM.md +9 -0
- data/vendor/crispasr/VERSION +1 -0
- data/vendor/crispasr/ggml/AUTHORS +335 -0
- data/vendor/crispasr/ggml/CMakeLists.txt +512 -0
- data/vendor/crispasr/ggml/LICENSE +21 -0
- data/vendor/crispasr/ggml/README.md +49 -0
- data/vendor/crispasr/ggml/cmake/FindNCCL.cmake +36 -0
- data/vendor/crispasr/ggml/cmake/GitVars.cmake +22 -0
- data/vendor/crispasr/ggml/cmake/common.cmake +50 -0
- data/vendor/crispasr/ggml/cmake/ggml-config.cmake.in +191 -0
- data/vendor/crispasr/ggml/ggml.pc.in +10 -0
- data/vendor/crispasr/ggml/include/ggml-alloc.h +85 -0
- data/vendor/crispasr/ggml/include/ggml-backend.h +431 -0
- data/vendor/crispasr/ggml/include/ggml-blas.h +25 -0
- data/vendor/crispasr/ggml/include/ggml-cann.h +123 -0
- data/vendor/crispasr/ggml/include/ggml-cpp.h +39 -0
- data/vendor/crispasr/ggml/include/ggml-cpu.h +151 -0
- data/vendor/crispasr/ggml/include/ggml-cuda.h +50 -0
- data/vendor/crispasr/ggml/include/ggml-hexagon.h +19 -0
- data/vendor/crispasr/ggml/include/ggml-metal.h +61 -0
- data/vendor/crispasr/ggml/include/ggml-opencl.h +26 -0
- data/vendor/crispasr/ggml/include/ggml-openvino.h +37 -0
- data/vendor/crispasr/ggml/include/ggml-opt.h +256 -0
- data/vendor/crispasr/ggml/include/ggml-rpc.h +35 -0
- data/vendor/crispasr/ggml/include/ggml-sycl.h +49 -0
- data/vendor/crispasr/ggml/include/ggml-virtgpu.h +14 -0
- data/vendor/crispasr/ggml/include/ggml-vulkan.h +29 -0
- data/vendor/crispasr/ggml/include/ggml-webgpu.h +19 -0
- data/vendor/crispasr/ggml/include/ggml-zdnn.h +17 -0
- data/vendor/crispasr/ggml/include/ggml-zendnn.h +22 -0
- data/vendor/crispasr/ggml/include/ggml.h +2887 -0
- data/vendor/crispasr/ggml/include/gguf.h +204 -0
- data/vendor/crispasr/ggml/src/CMakeLists.txt +493 -0
- data/vendor/crispasr/ggml/src/ggml-alloc.c +1323 -0
- data/vendor/crispasr/ggml/src/ggml-backend-dl.cpp +48 -0
- data/vendor/crispasr/ggml/src/ggml-backend-dl.h +44 -0
- data/vendor/crispasr/ggml/src/ggml-backend-impl.h +275 -0
- data/vendor/crispasr/ggml/src/ggml-backend-meta.cpp +2145 -0
- data/vendor/crispasr/ggml/src/ggml-backend-reg.cpp +586 -0
- data/vendor/crispasr/ggml/src/ggml-backend.cpp +2437 -0
- data/vendor/crispasr/ggml/src/ggml-common.h +1900 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/CMakeLists.txt +718 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/amx/amx.cpp +249 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/amx/amx.h +8 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/amx/common.h +115 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/amx/mmq.cpp +2512 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/amx/mmq.h +10 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp +98 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/arm/quants.c +4244 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/arm/repack.cpp +5156 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/loongarch/quants.c +2158 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp +82 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/powerpc/quants.c +2304 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp +38 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/riscv/quants.c +4455 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/riscv/repack.cpp +1703 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp +50 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/s390/quants.c +1465 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/wasm/quants.c +1220 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp +327 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/x86/quants.c +3970 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch/x86/repack.cpp +6407 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/arch-fallback.h +349 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/binary-ops.cpp +154 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/binary-ops.h +16 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/cmake/FindSIMD.cmake +100 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/common.h +95 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/ggml-cpu-impl.h +539 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/ggml-cpu.c +3791 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/ggml-cpu.cpp +703 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/hbm.cpp +55 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/hbm.h +8 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kernels.cpp +939 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kernels.h +90 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +1513 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/kleidiai/kleidiai.h +17 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/llamafile/sgemm.cpp +4051 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/llamafile/sgemm.h +25 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/ops.cpp +11662 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/ops.h +121 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/quants.c +1288 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/quants.h +103 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/repack.cpp +4836 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/repack.h +245 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/simd-gemm.h +226 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/simd-mappings.h +1329 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime.cpp +1025 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime.h +13 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp +3196 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/spacemit/ime_kernels.h +26 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/traits.cpp +36 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/traits.h +38 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/unary-ops.cpp +336 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/unary-ops.h +35 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/vec.cpp +681 -0
- data/vendor/crispasr/ggml/src/ggml-cpu/vec.h +1606 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/CMakeLists.txt +272 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/acc.cu +61 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/acc.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/add-id.cu +58 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/add-id.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/arange.cu +34 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/arange.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/argmax.cu +91 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/argmax.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/argsort.cu +265 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/argsort.cuh +19 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/binbcast.cu +534 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/binbcast.cuh +12 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/clamp.cu +45 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/clamp.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/col2im-1d.cu +81 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/col2im-1d.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/common.cuh +1489 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/concat.cu +204 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/concat.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/conv-transpose-1d.cu +97 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/conv-transpose-1d.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-dw.cu +161 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-dw.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-transpose.cu +115 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/conv2d-transpose.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/conv2d.cu +166 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/conv2d.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/convert.cu +892 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/convert.cuh +66 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/count-equal.cu +64 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/count-equal.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/cp-async.cuh +57 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/cpy-utils.cuh +217 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/cpy.cu +581 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/cpy.cuh +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/cross-entropy-loss.cu +177 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/cross-entropy-loss.cuh +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/cumsum.cu +307 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/cumsum.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/dequantize.cuh +99 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/diag.cu +77 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/diag.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/diagmask.cu +40 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/diagmask.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn-common.cuh +1212 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn-mma-f16.cuh +1860 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn-tile.cu +57 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn-tile.cuh +1309 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn-vec.cuh +600 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn-wmma-f16.cu +696 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn-wmma-f16.cuh +51 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn.cu +620 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fattn.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fill.cu +37 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/fill.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/gated_delta_net.cu +273 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/gated_delta_net.cuh +4 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/getrows.cu +332 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/getrows.cuh +15 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/ggml-cuda.cu +5580 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/gla.cu +93 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/gla.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/im2col.cu +274 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/im2col.cuh +6 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mean.cu +75 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mean.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mma.cuh +1333 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmf.cu +191 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmf.cuh +908 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmid.cu +164 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmid.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmq.cu +372 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmq.cuh +4175 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmvf.cu +862 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmvf.cuh +14 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmvq.cu +1161 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/mmvq.cuh +16 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/norm.cu +756 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/norm.cuh +20 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-adamw.cu +78 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-adamw.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-sgd.cu +49 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/opt-step-sgd.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/out-prod.cu +68 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/out-prod.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/pad.cu +106 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/pad.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/pad_reflect_1d.cu +91 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/pad_reflect_1d.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/pool2d.cu +94 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/pool2d.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/quantize.cu +443 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/quantize.cuh +41 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/reduce_rows.cuh +39 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/roll.cu +67 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/roll.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/rope.cu +665 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/rope.cuh +9 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/scale.cu +34 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/scale.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/set-rows.cu +330 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/set-rows.cuh +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/set.cu +39 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/set.cuh +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/softcap.cu +34 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/softcap.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/softmax.cu +472 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/softmax.cuh +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/solve_tri.cu +275 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/solve_tri.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/ssm-conv.cu +197 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/ssm-conv.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/ssm-scan.cu +342 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/ssm-scan.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/sum.cu +41 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/sum.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/sumrows.cu +43 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/sumrows.cuh +4 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu +6 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu +11 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu +10 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu +10 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu +12 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu +6 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu +12 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu +11 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu +10 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu +10 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu +10 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu +12 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu +11 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu +10 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu +10 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu +10 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu +12 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu +11 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu +7 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/top-k.cu +95 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/top-k.cuh +3 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/topk-moe.cu +415 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/topk-moe.cuh +27 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/tri.cu +136 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/tri.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/tsembd.cu +47 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/tsembd.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/unary.cu +644 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/unary.cuh +116 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/upscale.cu +293 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/upscale.cuh +5 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/vecdotq.cuh +1317 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/vendors/cuda.h +28 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/vendors/hip.h +306 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/vendors/musa.h +148 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/wkv.cu +199 -0
- data/vendor/crispasr/ggml/src/ggml-cuda/wkv.cuh +7 -0
- data/vendor/crispasr/ggml/src/ggml-impl.h +783 -0
- data/vendor/crispasr/ggml/src/ggml-metal/CMakeLists.txt +124 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-common.cpp +457 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-common.h +52 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-context.h +41 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-context.m +778 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-device.cpp +2117 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-device.h +299 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-device.m +2093 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-impl.h +1267 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-ops.cpp +5023 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal-ops.h +111 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal.cpp +954 -0
- data/vendor/crispasr/ggml/src/ggml-metal/ggml-metal.metal +11756 -0
- data/vendor/crispasr/ggml/src/ggml-opt.cpp +1094 -0
- data/vendor/crispasr/ggml/src/ggml-quants.c +5491 -0
- data/vendor/crispasr/ggml/src/ggml-quants.h +112 -0
- data/vendor/crispasr/ggml/src/ggml-threading.cpp +12 -0
- data/vendor/crispasr/ggml/src/ggml-threading.h +14 -0
- data/vendor/crispasr/ggml/src/ggml.c +7925 -0
- data/vendor/crispasr/ggml/src/ggml.cpp +26 -0
- data/vendor/crispasr/ggml/src/gguf.cpp +1556 -0
- data/vendor/crispasr/src/cohere-arch.h +137 -0
- data/vendor/crispasr/src/cohere.cpp +5642 -0
- data/vendor/crispasr/src/cohere.h +327 -0
- data/vendor/crispasr/src/cohere_batch_planner.h +82 -0
- data/vendor/crispasr/src/cohere_chunking.h +64 -0
- data/vendor/crispasr/src/cohere_decoder_batch_layout.h +60 -0
- data/vendor/crispasr/src/cohere_encoder_padded_layout.h +27 -0
- data/vendor/crispasr/src/cohere_frontend.cpp +189 -0
- data/vendor/crispasr/src/cohere_frontend.h +31 -0
- data/vendor/crispasr/src/cohere_ragged_controller.h +149 -0
- data/vendor/crispasr/src/cohere_token_renderer.h +181 -0
- data/vendor/crispasr/src/core/attention.h +924 -0
- data/vendor/crispasr/src/core/audio_chunking.h +97 -0
- data/vendor/crispasr/src/core/beam_decode.h +486 -0
- data/vendor/crispasr/src/core/cpu_ops.h +135 -0
- data/vendor/crispasr/src/core/gguf_loader.cpp +1021 -0
- data/vendor/crispasr/src/core/gguf_loader.h +216 -0
- data/vendor/crispasr/src/core/gpu_backend_pref.h +119 -0
- data/vendor/crispasr/src/core/mel.cpp +519 -0
- data/vendor/crispasr/src/core/mel.h +265 -0
- data/vendor/crispasr/src/core/ngram_loop_fix.h +173 -0
- data/vendor/crispasr/src/core/repetition_loop_guard.h +54 -0
- data/vendor/crispasr/src/crispasr_imatrix.cpp +255 -0
- data/vendor/crispasr/src/crispasr_imatrix.h +38 -0
- metadata +596 -0
|
@@ -0,0 +1,727 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "fiddle"
|
|
5
|
+
require "json"
|
|
6
|
+
require "securerandom"
|
|
7
|
+
|
|
8
|
+
module Cohere
|
|
9
|
+
module Transcribe
|
|
10
|
+
module State
|
|
11
|
+
STATE_SCHEMA_VERSION = 1
|
|
12
|
+
STATE_SUFFIX = ".cohere-transcribe.manifest.json"
|
|
13
|
+
CHECKPOINT_SUFFIX = ".cohere-transcribe.asr.json"
|
|
14
|
+
MAX_STATE_BYTES = 64 * 1024 * 1024
|
|
15
|
+
DEFERRED_PUBLICATION_EXCEPTIONS = {
|
|
16
|
+
SignalException => :never,
|
|
17
|
+
SystemExit => :never
|
|
18
|
+
}.freeze
|
|
19
|
+
|
|
20
|
+
DirectoryBinding = Data.define(
|
|
21
|
+
:access_path,
|
|
22
|
+
:canonical_path,
|
|
23
|
+
:device,
|
|
24
|
+
:inode
|
|
25
|
+
) do
|
|
26
|
+
def self.capture(path)
|
|
27
|
+
access_path = Pathname(path).expand_path.cleanpath
|
|
28
|
+
canonical_path = access_path.realpath
|
|
29
|
+
stat = canonical_path.lstat
|
|
30
|
+
unless stat.directory? && !stat.symlink?
|
|
31
|
+
raise TranscriptionRuntimeError, "Publication parent is not a real directory: #{access_path}"
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
new(
|
|
35
|
+
access_path: access_path.freeze,
|
|
36
|
+
canonical_path: canonical_path.freeze,
|
|
37
|
+
device: stat.dev,
|
|
38
|
+
inode: stat.ino
|
|
39
|
+
)
|
|
40
|
+
rescue SystemCallError, ArgumentError => e
|
|
41
|
+
raise TranscriptionRuntimeError,
|
|
42
|
+
"Cannot bind publication parent #{path}: #{e.message}"
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def verify!
|
|
46
|
+
resolved = access_path.realpath
|
|
47
|
+
stat = canonical_path.lstat
|
|
48
|
+
valid = resolved == canonical_path && stat.directory? && !stat.symlink? &&
|
|
49
|
+
stat.dev == device && stat.ino == inode
|
|
50
|
+
return self if valid
|
|
51
|
+
|
|
52
|
+
raise TranscriptionRuntimeError,
|
|
53
|
+
"Publication parent changed after planning: #{access_path}"
|
|
54
|
+
rescue SystemCallError, ArgumentError => e
|
|
55
|
+
raise TranscriptionRuntimeError,
|
|
56
|
+
"Publication parent changed after planning: #{access_path} (#{e.message})"
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# A retained directory descriptor is the authority for publication I/O.
|
|
61
|
+
# Path checks alone cannot close a rename/symlink TOCTOU window: after
|
|
62
|
+
# this object opens and verifies the planned inode, every entry operation
|
|
63
|
+
# is performed with the POSIX *at family against that descriptor.
|
|
64
|
+
class BoundDirectory
|
|
65
|
+
AT_FUNCTION_SIGNATURES = {
|
|
66
|
+
openat: [[Fiddle::TYPE_INT, Fiddle::TYPE_VOIDP, Fiddle::TYPE_INT, Fiddle::TYPE_INT], Fiddle::TYPE_INT],
|
|
67
|
+
renameat: [
|
|
68
|
+
[Fiddle::TYPE_INT, Fiddle::TYPE_VOIDP, Fiddle::TYPE_INT, Fiddle::TYPE_VOIDP],
|
|
69
|
+
Fiddle::TYPE_INT
|
|
70
|
+
],
|
|
71
|
+
unlinkat: [[Fiddle::TYPE_INT, Fiddle::TYPE_VOIDP, Fiddle::TYPE_INT], Fiddle::TYPE_INT],
|
|
72
|
+
mkdirat: [[Fiddle::TYPE_INT, Fiddle::TYPE_VOIDP, Fiddle::TYPE_INT], Fiddle::TYPE_INT]
|
|
73
|
+
}.freeze
|
|
74
|
+
|
|
75
|
+
class << self
|
|
76
|
+
def open(binding, guards: [binding])
|
|
77
|
+
ensure_supported!
|
|
78
|
+
succeeded = false
|
|
79
|
+
guards = guards.uniq.freeze
|
|
80
|
+
guards.each(&:verify!)
|
|
81
|
+
flags = File::RDONLY | File::NONBLOCK | File.const_get(:NOFOLLOW)
|
|
82
|
+
descriptor = IO.sysopen(binding.canonical_path.to_s, flags)
|
|
83
|
+
handle = File.new(descriptor, "rb", autoclose: true)
|
|
84
|
+
descriptor = nil
|
|
85
|
+
handle.close_on_exec = true
|
|
86
|
+
stat = handle.stat
|
|
87
|
+
unless stat.directory? && stat.dev == binding.device && stat.ino == binding.inode
|
|
88
|
+
raise TranscriptionRuntimeError,
|
|
89
|
+
"Publication parent changed while it was being opened: #{binding.access_path}"
|
|
90
|
+
end
|
|
91
|
+
guards.each(&:verify!)
|
|
92
|
+
bound = new(binding, handle, guards)
|
|
93
|
+
succeeded = true
|
|
94
|
+
bound
|
|
95
|
+
rescue Interrupt, SystemExit
|
|
96
|
+
raise
|
|
97
|
+
rescue SystemCallError, ArgumentError => e
|
|
98
|
+
raise TranscriptionRuntimeError,
|
|
99
|
+
"Cannot open planned publication parent #{binding.access_path}: #{e.message}"
|
|
100
|
+
ensure
|
|
101
|
+
handle&.close if handle && !succeeded
|
|
102
|
+
IO.new(descriptor).close if descriptor
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def ensure_supported!
|
|
106
|
+
required_constants = %i[NOFOLLOW]
|
|
107
|
+
missing_constants = required_constants.reject { |name| File.const_defined?(name) }
|
|
108
|
+
missing_functions = AT_FUNCTION_SIGNATURES.keys.reject do |name|
|
|
109
|
+
function(name)
|
|
110
|
+
true
|
|
111
|
+
rescue Fiddle::DLError
|
|
112
|
+
false
|
|
113
|
+
end
|
|
114
|
+
return if missing_constants.empty? && missing_functions.empty?
|
|
115
|
+
|
|
116
|
+
details = (missing_constants + missing_functions).join(", ")
|
|
117
|
+
raise TranscriptionRuntimeError,
|
|
118
|
+
"Descriptor-relative publication is unavailable on this platform (missing #{details})"
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def function(name)
|
|
122
|
+
@function_guard ||= Mutex.new
|
|
123
|
+
@functions ||= {}
|
|
124
|
+
@function_guard.synchronize do
|
|
125
|
+
@functions[name] ||= begin
|
|
126
|
+
arguments, result = AT_FUNCTION_SIGNATURES.fetch(name)
|
|
127
|
+
address = Fiddle::Handle::DEFAULT[name.to_s]
|
|
128
|
+
Fiddle::Function.new(address, arguments, result)
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
attr_reader :binding
|
|
135
|
+
|
|
136
|
+
def initialize(binding, handle, guards)
|
|
137
|
+
@binding = binding
|
|
138
|
+
@handle = handle
|
|
139
|
+
@guards = guards
|
|
140
|
+
@closed = false
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def create_temporary(basename, suffix)
|
|
144
|
+
validate_entry_name!(basename)
|
|
145
|
+
100.times do
|
|
146
|
+
name = ".#{basename}.#{SecureRandom.hex(12)}#{suffix}"
|
|
147
|
+
begin
|
|
148
|
+
handle = open_entry(
|
|
149
|
+
name,
|
|
150
|
+
File::WRONLY | File::CREAT | File::EXCL | File::NOFOLLOW,
|
|
151
|
+
0o600,
|
|
152
|
+
mode: "wb"
|
|
153
|
+
)
|
|
154
|
+
return [name.freeze, handle]
|
|
155
|
+
rescue Errno::EEXIST
|
|
156
|
+
next
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
raise TranscriptionRuntimeError,
|
|
160
|
+
"Cannot allocate a unique publication temporary for #{basename}"
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def open_regular(name, writable: false)
|
|
164
|
+
flags = writable ? File::RDWR : File::RDONLY
|
|
165
|
+
flags |= File::NOFOLLOW | File::NONBLOCK
|
|
166
|
+
handle = open_entry(name, flags, 0, mode: writable ? "r+b" : "rb")
|
|
167
|
+
unless handle.stat.file?
|
|
168
|
+
handle.close
|
|
169
|
+
raise TranscriptionRuntimeError,
|
|
170
|
+
"Publication entry is not a regular file: #{display_path(name)}"
|
|
171
|
+
end
|
|
172
|
+
handle
|
|
173
|
+
rescue Errno::ELOOP, Errno::EISDIR, Errno::ENXIO => e
|
|
174
|
+
raise TranscriptionRuntimeError,
|
|
175
|
+
"Publication entry is not a regular file: #{display_path(name)}",
|
|
176
|
+
cause: e
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def regular_entry?(name)
|
|
180
|
+
handle = nil
|
|
181
|
+
Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
|
|
182
|
+
handle = open_regular(name)
|
|
183
|
+
end
|
|
184
|
+
true
|
|
185
|
+
rescue Errno::ENOENT
|
|
186
|
+
false
|
|
187
|
+
ensure
|
|
188
|
+
handle&.close
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def same_regular_entry?(name, expected_stat)
|
|
192
|
+
handle = nil
|
|
193
|
+
Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
|
|
194
|
+
handle = open_regular(name)
|
|
195
|
+
end
|
|
196
|
+
current = handle.stat
|
|
197
|
+
current.dev == expected_stat.dev && current.ino == expected_stat.ino
|
|
198
|
+
rescue Errno::ENOENT
|
|
199
|
+
false
|
|
200
|
+
ensure
|
|
201
|
+
handle&.close
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def rename(source, destination)
|
|
205
|
+
validate_entry_name!(source)
|
|
206
|
+
validate_entry_name!(destination)
|
|
207
|
+
call_at!(:renameat, descriptor, source, descriptor, destination)
|
|
208
|
+
nil
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def mkdir(name, mode = 0o777)
|
|
212
|
+
validate_entry_name!(name)
|
|
213
|
+
call_at!(:mkdirat, descriptor, name, mode)
|
|
214
|
+
nil
|
|
215
|
+
rescue Errno::EEXIST
|
|
216
|
+
nil
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def open_child_directory(name, access_path:, canonical_path:)
|
|
220
|
+
validate_entry_name!(name)
|
|
221
|
+
handle = open_entry(
|
|
222
|
+
name,
|
|
223
|
+
File::RDONLY | File::NONBLOCK | File::NOFOLLOW,
|
|
224
|
+
0,
|
|
225
|
+
mode: "rb"
|
|
226
|
+
)
|
|
227
|
+
stat = handle.stat
|
|
228
|
+
unless stat.directory?
|
|
229
|
+
raise TranscriptionRuntimeError,
|
|
230
|
+
"Publication directory component is not a real directory: #{access_path}"
|
|
231
|
+
end
|
|
232
|
+
child_binding = DirectoryBinding.new(
|
|
233
|
+
access_path: Pathname(access_path).freeze,
|
|
234
|
+
canonical_path: Pathname(canonical_path).freeze,
|
|
235
|
+
device: stat.dev,
|
|
236
|
+
inode: stat.ino
|
|
237
|
+
)
|
|
238
|
+
child = self.class.new(child_binding, handle, (@guards + [child_binding]).uniq.freeze)
|
|
239
|
+
child.verify!
|
|
240
|
+
handle = nil
|
|
241
|
+
child
|
|
242
|
+
rescue Errno::ELOOP, Errno::EISDIR, Errno::ENXIO => e
|
|
243
|
+
raise TranscriptionRuntimeError,
|
|
244
|
+
"Publication directory component is not a real directory: #{access_path}",
|
|
245
|
+
cause: e
|
|
246
|
+
ensure
|
|
247
|
+
handle&.close
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
def unlink(name, missing_ok: false)
|
|
251
|
+
validate_entry_name!(name)
|
|
252
|
+
call_at!(:unlinkat, descriptor, name, 0)
|
|
253
|
+
nil
|
|
254
|
+
rescue Errno::ENOENT
|
|
255
|
+
raise unless missing_ok
|
|
256
|
+
|
|
257
|
+
nil
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
def verify!
|
|
261
|
+
stat = @handle.stat
|
|
262
|
+
unless stat.directory? && stat.dev == binding.device && stat.ino == binding.inode
|
|
263
|
+
raise TranscriptionRuntimeError,
|
|
264
|
+
"Retained publication parent changed identity: #{binding.access_path}"
|
|
265
|
+
end
|
|
266
|
+
@guards.each(&:verify!)
|
|
267
|
+
self
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
def fsync
|
|
271
|
+
@handle.fsync
|
|
272
|
+
rescue Errno::EACCES, Errno::EBADF, Errno::EINVAL, Errno::EISDIR,
|
|
273
|
+
Errno::ENOTSUP, Errno::EPERM
|
|
274
|
+
nil
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
def close
|
|
278
|
+
return if @closed
|
|
279
|
+
|
|
280
|
+
@handle.close
|
|
281
|
+
@closed = true
|
|
282
|
+
nil
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
def display_path(name)
|
|
286
|
+
binding.canonical_path.join(name)
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
private
|
|
290
|
+
|
|
291
|
+
def descriptor
|
|
292
|
+
@handle.fileno
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
def open_entry(name, flags, permissions, mode:)
|
|
296
|
+
validate_entry_name!(name)
|
|
297
|
+
file_descriptor = call_at!(:openat, descriptor, name, flags, permissions)
|
|
298
|
+
handle = File.new(file_descriptor, mode, autoclose: true)
|
|
299
|
+
file_descriptor = nil
|
|
300
|
+
handle.close_on_exec = true
|
|
301
|
+
handle
|
|
302
|
+
ensure
|
|
303
|
+
IO.new(file_descriptor).close if file_descriptor
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
def call_at!(name, *)
|
|
307
|
+
result = self.class.function(name).call(*)
|
|
308
|
+
return result unless result == -1
|
|
309
|
+
|
|
310
|
+
error_number = Fiddle.last_error
|
|
311
|
+
raise SystemCallError.new("#{name} #{binding.canonical_path}", error_number)
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
def validate_entry_name!(name)
|
|
315
|
+
value = name.to_s
|
|
316
|
+
return if !value.empty? && value != "." && value != ".." &&
|
|
317
|
+
!value.include?(File::SEPARATOR) && !value.include?("\0")
|
|
318
|
+
|
|
319
|
+
raise ArgumentError, "Invalid descriptor-relative publication name: #{value.inspect}"
|
|
320
|
+
end
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
SourceSnapshot = Data.define(
|
|
324
|
+
:canonical_path,
|
|
325
|
+
:device,
|
|
326
|
+
:inode,
|
|
327
|
+
:size,
|
|
328
|
+
:mtime_ns,
|
|
329
|
+
:ctime_ns
|
|
330
|
+
) do
|
|
331
|
+
def self.capture(path)
|
|
332
|
+
canonical = Pathname(path).realpath
|
|
333
|
+
stat = canonical.stat
|
|
334
|
+
new(
|
|
335
|
+
canonical_path: canonical.to_s.freeze,
|
|
336
|
+
device: stat.dev,
|
|
337
|
+
inode: stat.ino,
|
|
338
|
+
size: stat.size,
|
|
339
|
+
mtime_ns: State.time_nanoseconds(stat.mtime),
|
|
340
|
+
ctime_ns: State.time_nanoseconds(stat.ctime)
|
|
341
|
+
)
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
def payload
|
|
345
|
+
{
|
|
346
|
+
"canonical_path" => canonical_path,
|
|
347
|
+
"snapshot" => {
|
|
348
|
+
"device" => device,
|
|
349
|
+
"inode" => inode,
|
|
350
|
+
"size" => size,
|
|
351
|
+
"mtime_ns" => mtime_ns,
|
|
352
|
+
"ctime_ns" => ctime_ns
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
end
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
module_function
|
|
359
|
+
|
|
360
|
+
def with_bound_parent(path, directory_binding: nil, guard_bindings: nil)
|
|
361
|
+
path = Pathname(path).expand_path
|
|
362
|
+
directory_binding ||= DirectoryBinding.capture(path.dirname)
|
|
363
|
+
unless [directory_binding.access_path, directory_binding.canonical_path].include?(path.dirname)
|
|
364
|
+
raise TranscriptionRuntimeError,
|
|
365
|
+
"Publication path is outside its planned parent: #{path}"
|
|
366
|
+
end
|
|
367
|
+
guards = Array(guard_bindings).compact
|
|
368
|
+
guards << directory_binding
|
|
369
|
+
bound = nil
|
|
370
|
+
Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
|
|
371
|
+
bound = BoundDirectory.open(directory_binding, guards: guards.uniq)
|
|
372
|
+
end
|
|
373
|
+
yield bound, path.basename.to_s
|
|
374
|
+
ensure
|
|
375
|
+
bound&.close
|
|
376
|
+
end
|
|
377
|
+
|
|
378
|
+
def ensure_bound_directory(path, root_binding: nil, operation_hook: nil)
|
|
379
|
+
target = Pathname(path).expand_path.cleanpath
|
|
380
|
+
bindings = []
|
|
381
|
+
current = nil
|
|
382
|
+
if root_binding
|
|
383
|
+
root_binding.verify!
|
|
384
|
+
relative = target.relative_path_from(root_binding.canonical_path)
|
|
385
|
+
if relative.each_filename.first == ".."
|
|
386
|
+
raise TranscriptionRuntimeError,
|
|
387
|
+
"Publication directory escapes its bound root: #{target}"
|
|
388
|
+
end
|
|
389
|
+
current_binding = root_binding
|
|
390
|
+
access_cursor = root_binding.canonical_path
|
|
391
|
+
canonical_cursor = root_binding.canonical_path
|
|
392
|
+
components = relative.each_filename.reject { |name| name == "." }
|
|
393
|
+
else
|
|
394
|
+
missing = []
|
|
395
|
+
cursor = target
|
|
396
|
+
until cursor.exist? || cursor.symlink?
|
|
397
|
+
parent = cursor.dirname
|
|
398
|
+
raise TranscriptionRuntimeError, "Cannot resolve publication directory #{target}" if parent == cursor
|
|
399
|
+
|
|
400
|
+
missing.unshift(cursor.basename.to_s)
|
|
401
|
+
cursor = parent
|
|
402
|
+
end
|
|
403
|
+
current_binding = DirectoryBinding.capture(cursor)
|
|
404
|
+
access_cursor = current_binding.access_path
|
|
405
|
+
canonical_cursor = current_binding.canonical_path
|
|
406
|
+
components = missing
|
|
407
|
+
end
|
|
408
|
+
|
|
409
|
+
Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
|
|
410
|
+
current = BoundDirectory.open(current_binding, guards: [current_binding])
|
|
411
|
+
bindings << current
|
|
412
|
+
end
|
|
413
|
+
operation_hook&.call(:ancestor_opened, current.binding.access_path)
|
|
414
|
+
components.each do |component|
|
|
415
|
+
operation_hook&.call(:before_mkdir, access_cursor.join(component))
|
|
416
|
+
current.mkdir(component)
|
|
417
|
+
operation_hook&.call(:after_mkdir, access_cursor.join(component))
|
|
418
|
+
access_cursor = access_cursor.join(component)
|
|
419
|
+
canonical_cursor = canonical_cursor.join(component)
|
|
420
|
+
Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
|
|
421
|
+
current = current.open_child_directory(
|
|
422
|
+
component,
|
|
423
|
+
access_path: access_cursor,
|
|
424
|
+
canonical_path: canonical_cursor
|
|
425
|
+
)
|
|
426
|
+
bindings << current
|
|
427
|
+
end
|
|
428
|
+
end
|
|
429
|
+
bindings.each(&:verify!)
|
|
430
|
+
current.binding
|
|
431
|
+
rescue ArgumentError => e
|
|
432
|
+
raise TranscriptionRuntimeError, "Cannot bind publication directory #{path}: #{e.message}"
|
|
433
|
+
ensure
|
|
434
|
+
bindings&.reverse_each(&:close)
|
|
435
|
+
end
|
|
436
|
+
|
|
437
|
+
def time_nanoseconds(time)
|
|
438
|
+
(time.to_i * 1_000_000_000) + time.nsec
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
def output_parent_and_stem(output_paths)
|
|
442
|
+
raise ArgumentError, "An output set must contain at least one path" if output_paths.empty?
|
|
443
|
+
|
|
444
|
+
paths = output_paths.values.map { |path| Pathname(path) }
|
|
445
|
+
parent = paths.first.dirname
|
|
446
|
+
stem = paths.first.basename(paths.first.extname).to_s
|
|
447
|
+
unless paths.drop(1).all? do |path|
|
|
448
|
+
path.dirname == parent && path.basename(path.extname).to_s == stem
|
|
449
|
+
end
|
|
450
|
+
raise ArgumentError, "All formats in one output set must share a parent and stem"
|
|
451
|
+
end
|
|
452
|
+
|
|
453
|
+
[parent, stem]
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
def state_path_for_outputs(output_paths)
|
|
457
|
+
parent, stem = output_parent_and_stem(output_paths)
|
|
458
|
+
parent.join(".#{stem}#{STATE_SUFFIX}")
|
|
459
|
+
end
|
|
460
|
+
|
|
461
|
+
def checkpoint_path_for_outputs(output_paths)
|
|
462
|
+
parent, stem = output_parent_and_stem(output_paths)
|
|
463
|
+
parent.join(".#{stem}#{CHECKPOINT_SUFFIX}")
|
|
464
|
+
end
|
|
465
|
+
|
|
466
|
+
def canonical_json(value)
|
|
467
|
+
JSON.generate(deep_sort(value))
|
|
468
|
+
end
|
|
469
|
+
|
|
470
|
+
def envelope(payload)
|
|
471
|
+
{
|
|
472
|
+
"schema_version" => STATE_SCHEMA_VERSION,
|
|
473
|
+
"payload_sha256" => Digest::SHA256.hexdigest(canonical_json(payload)),
|
|
474
|
+
"payload" => payload
|
|
475
|
+
}
|
|
476
|
+
end
|
|
477
|
+
|
|
478
|
+
def decode_state(path, directory_binding: nil, guard_bindings: nil)
|
|
479
|
+
path = Pathname(path)
|
|
480
|
+
contents = securely_read_regular_file(
|
|
481
|
+
path,
|
|
482
|
+
directory_binding: directory_binding,
|
|
483
|
+
guard_bindings: guard_bindings
|
|
484
|
+
)
|
|
485
|
+
decoded = JSON.parse(contents)
|
|
486
|
+
return [nil, "state marker root is not an object"] unless decoded.is_a?(Hash)
|
|
487
|
+
return [nil, "state marker schema is unsupported"] unless decoded["schema_version"] == STATE_SCHEMA_VERSION
|
|
488
|
+
|
|
489
|
+
payload = decoded["payload"]
|
|
490
|
+
return [nil, "state marker payload is not an object"] unless payload.is_a?(Hash)
|
|
491
|
+
|
|
492
|
+
expected = Digest::SHA256.hexdigest(canonical_json(payload))
|
|
493
|
+
return [nil, "state marker integrity check failed"] unless secure_digest_equal?(decoded["payload_sha256"], expected)
|
|
494
|
+
|
|
495
|
+
[payload, nil]
|
|
496
|
+
rescue Errno::ENOENT
|
|
497
|
+
[nil, "state marker is missing or not a regular file"]
|
|
498
|
+
rescue JSON::ParserError, JSON::GeneratorError, EncodingError, SystemCallError,
|
|
499
|
+
TranscriptionRuntimeError, TypeError, ArgumentError => e
|
|
500
|
+
[nil, "state marker is unreadable (#{e.class}: #{e.message})"]
|
|
501
|
+
end
|
|
502
|
+
|
|
503
|
+
def write_state_atomic(path, payload, source_snapshot:, directory_binding: nil,
|
|
504
|
+
guard_bindings: nil, operation_hook: nil, rename: nil)
|
|
505
|
+
path = Pathname(path).expand_path
|
|
506
|
+
directory_binding ||= ensure_bound_directory(path.dirname)
|
|
507
|
+
with_bound_parent(
|
|
508
|
+
path,
|
|
509
|
+
directory_binding: directory_binding,
|
|
510
|
+
guard_bindings: guard_bindings
|
|
511
|
+
) do |bound, basename|
|
|
512
|
+
operation_hook&.call(:directory_opened, path)
|
|
513
|
+
temporary_name = nil
|
|
514
|
+
temporary = nil
|
|
515
|
+
backup_name = nil
|
|
516
|
+
backup = nil
|
|
517
|
+
source = nil
|
|
518
|
+
committed = false
|
|
519
|
+
preserved_backup = false
|
|
520
|
+
primary_failed = false
|
|
521
|
+
begin
|
|
522
|
+
mode = bound_state_mode(bound, basename)
|
|
523
|
+
Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
|
|
524
|
+
temporary_name, temporary = bound.create_temporary(basename, ".tmp")
|
|
525
|
+
end
|
|
526
|
+
temporary.chmod(mode)
|
|
527
|
+
operation_hook&.call(:before_stage_write, path)
|
|
528
|
+
temporary.write(JSON.pretty_generate(envelope(payload)))
|
|
529
|
+
temporary.write("\n")
|
|
530
|
+
temporary.flush
|
|
531
|
+
temporary.fsync
|
|
532
|
+
temporary.close
|
|
533
|
+
temporary = nil
|
|
534
|
+
bound.verify!
|
|
535
|
+
|
|
536
|
+
operation_hook&.call(:before_backup_open, path)
|
|
537
|
+
Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
|
|
538
|
+
source = begin
|
|
539
|
+
bound.open_regular(basename)
|
|
540
|
+
rescue Errno::ENOENT
|
|
541
|
+
nil
|
|
542
|
+
end
|
|
543
|
+
end
|
|
544
|
+
if source
|
|
545
|
+
Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
|
|
546
|
+
backup_name, backup = bound.create_temporary(basename, ".bak")
|
|
547
|
+
end
|
|
548
|
+
begin
|
|
549
|
+
backup.chmod(source.stat.mode & 0o7777)
|
|
550
|
+
operation_hook&.call(:before_backup_copy, path)
|
|
551
|
+
IO.copy_stream(source, backup)
|
|
552
|
+
backup.flush
|
|
553
|
+
backup.fsync
|
|
554
|
+
ensure
|
|
555
|
+
source.close
|
|
556
|
+
backup.close
|
|
557
|
+
end
|
|
558
|
+
end
|
|
559
|
+
|
|
560
|
+
ensure_source_unchanged!(source_snapshot)
|
|
561
|
+
bound.verify!
|
|
562
|
+
operation_hook&.call(:before_rename, path)
|
|
563
|
+
Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
|
|
564
|
+
if rename
|
|
565
|
+
committed = true
|
|
566
|
+
rename.call(bound, temporary_name, basename, :commit)
|
|
567
|
+
else
|
|
568
|
+
bound.rename(temporary_name, basename)
|
|
569
|
+
committed = true
|
|
570
|
+
end
|
|
571
|
+
end
|
|
572
|
+
operation_hook&.call(:after_rename, path)
|
|
573
|
+
bound.verify!
|
|
574
|
+
bound.fsync
|
|
575
|
+
path
|
|
576
|
+
rescue Exception => e # rubocop:disable Lint/RescueException -- rollback includes interrupts
|
|
577
|
+
primary_failed = true
|
|
578
|
+
if committed
|
|
579
|
+
begin
|
|
580
|
+
Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
|
|
581
|
+
bound.unlink(basename, missing_ok: true)
|
|
582
|
+
if backup_name
|
|
583
|
+
if rename
|
|
584
|
+
rename.call(bound, backup_name, basename, :rollback)
|
|
585
|
+
else
|
|
586
|
+
bound.rename(backup_name, basename)
|
|
587
|
+
end
|
|
588
|
+
end
|
|
589
|
+
backup_name = nil
|
|
590
|
+
end
|
|
591
|
+
bound.fsync
|
|
592
|
+
rescue SystemCallError, TranscriptionRuntimeError => rollback_error
|
|
593
|
+
retained = backup_name ? bound.display_path(backup_name) : nil
|
|
594
|
+
preserved_backup = !backup_name.nil?
|
|
595
|
+
raise TranscriptionRuntimeError,
|
|
596
|
+
"State commit failed and rollback was incomplete (#{rollback_error.message}); " \
|
|
597
|
+
"preserved backup: #{retained}",
|
|
598
|
+
cause: e
|
|
599
|
+
end
|
|
600
|
+
end
|
|
601
|
+
raise e
|
|
602
|
+
ensure
|
|
603
|
+
cleanup_failure = nil
|
|
604
|
+
cleanup_operations = [
|
|
605
|
+
-> { temporary&.close unless temporary&.closed? },
|
|
606
|
+
-> { backup&.close unless backup&.closed? },
|
|
607
|
+
-> { bound.unlink(temporary_name, missing_ok: true) if temporary_name },
|
|
608
|
+
lambda do
|
|
609
|
+
bound.unlink(backup_name, missing_ok: true) if backup_name && !preserved_backup
|
|
610
|
+
end
|
|
611
|
+
]
|
|
612
|
+
cleanup_operations.each do |operation|
|
|
613
|
+
operation.call
|
|
614
|
+
rescue SystemCallError, IOError => e
|
|
615
|
+
cleanup_failure ||= e
|
|
616
|
+
end
|
|
617
|
+
raise cleanup_failure if cleanup_failure && !primary_failed
|
|
618
|
+
end
|
|
619
|
+
end
|
|
620
|
+
end
|
|
621
|
+
|
|
622
|
+
def bound_state_mode(bound, basename)
|
|
623
|
+
existing = nil
|
|
624
|
+
Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
|
|
625
|
+
existing = bound.open_regular(basename)
|
|
626
|
+
end
|
|
627
|
+
existing.stat.mode & 0o7777
|
|
628
|
+
rescue Errno::ENOENT
|
|
629
|
+
current_umask = File.umask
|
|
630
|
+
File.umask(current_umask)
|
|
631
|
+
0o666 & ~current_umask
|
|
632
|
+
ensure
|
|
633
|
+
existing&.close
|
|
634
|
+
end
|
|
635
|
+
private_class_method :bound_state_mode
|
|
636
|
+
|
|
637
|
+
def ensure_source_unchanged!(snapshot)
|
|
638
|
+
return if SourceSnapshot.capture(snapshot.canonical_path) == snapshot
|
|
639
|
+
|
|
640
|
+
raise TranscriptionRuntimeError, "Source changed while processing: #{snapshot.canonical_path}"
|
|
641
|
+
rescue SystemCallError, ArgumentError => e
|
|
642
|
+
raise TranscriptionRuntimeError,
|
|
643
|
+
"Cannot re-check source #{snapshot.canonical_path}: #{e.message}"
|
|
644
|
+
end
|
|
645
|
+
|
|
646
|
+
def securely_read_regular_file(path, directory_binding: nil, guard_bindings: nil)
|
|
647
|
+
if directory_binding
|
|
648
|
+
return with_bound_parent(
|
|
649
|
+
path,
|
|
650
|
+
directory_binding: directory_binding,
|
|
651
|
+
guard_bindings: guard_bindings
|
|
652
|
+
) do |bound, basename|
|
|
653
|
+
bound.verify!
|
|
654
|
+
handle = nil
|
|
655
|
+
Thread.handle_interrupt(DEFERRED_PUBLICATION_EXCEPTIONS) do
|
|
656
|
+
handle = bound.open_regular(basename)
|
|
657
|
+
end
|
|
658
|
+
opened = handle.stat
|
|
659
|
+
raise TranscriptionRuntimeError, "State marker is too large: #{path}" if opened.size > MAX_STATE_BYTES
|
|
660
|
+
|
|
661
|
+
contents = handle.read(MAX_STATE_BYTES + 1)
|
|
662
|
+
raise TranscriptionRuntimeError, "State marker is too large: #{path}" if contents.bytesize > MAX_STATE_BYTES
|
|
663
|
+
unless bound.same_regular_entry?(basename, opened)
|
|
664
|
+
raise TranscriptionRuntimeError, "State marker changed while it was being read: #{path}"
|
|
665
|
+
end
|
|
666
|
+
|
|
667
|
+
bound.verify!
|
|
668
|
+
contents
|
|
669
|
+
ensure
|
|
670
|
+
handle&.close
|
|
671
|
+
end
|
|
672
|
+
end
|
|
673
|
+
|
|
674
|
+
before = path.lstat
|
|
675
|
+
raise TranscriptionRuntimeError, "State marker is not a regular file: #{path}" unless before.file? && !before.symlink?
|
|
676
|
+
raise TranscriptionRuntimeError, "State marker is too large: #{path}" if before.size > MAX_STATE_BYTES
|
|
677
|
+
|
|
678
|
+
flags = File::RDONLY
|
|
679
|
+
flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
|
|
680
|
+
flags |= File::CLOEXEC if defined?(File::CLOEXEC)
|
|
681
|
+
descriptor = ::IO.sysopen(path.to_s, flags)
|
|
682
|
+
handle = ::IO.new(descriptor, "rb", autoclose: true)
|
|
683
|
+
descriptor = nil
|
|
684
|
+
opened = handle.stat
|
|
685
|
+
current = path.lstat
|
|
686
|
+
unless opened.file? && same_file?(opened, before) && same_file?(opened, current)
|
|
687
|
+
raise TranscriptionRuntimeError, "State marker changed while it was being opened: #{path}"
|
|
688
|
+
end
|
|
689
|
+
|
|
690
|
+
handle.read(MAX_STATE_BYTES + 1).tap do |contents|
|
|
691
|
+
raise TranscriptionRuntimeError, "State marker is too large: #{path}" if contents.bytesize > MAX_STATE_BYTES
|
|
692
|
+
end
|
|
693
|
+
ensure
|
|
694
|
+
handle&.close
|
|
695
|
+
::IO.new(descriptor).close if descriptor
|
|
696
|
+
end
|
|
697
|
+
|
|
698
|
+
def same_file?(first, second)
|
|
699
|
+
first.dev == second.dev && first.ino == second.ino && first.file? && second.file?
|
|
700
|
+
end
|
|
701
|
+
|
|
702
|
+
def secure_digest_equal?(actual, expected)
|
|
703
|
+
return false unless actual.is_a?(String) && actual.bytesize == expected.bytesize
|
|
704
|
+
|
|
705
|
+
difference = 0
|
|
706
|
+
actual.bytes.zip(expected.bytes) { |left, right| difference |= left ^ right }
|
|
707
|
+
difference.zero?
|
|
708
|
+
end
|
|
709
|
+
|
|
710
|
+
def deep_sort(value)
|
|
711
|
+
case value
|
|
712
|
+
when Hash
|
|
713
|
+
value.keys.map(&:to_s).sort.to_h do |key|
|
|
714
|
+
original_key = value.key?(key) ? key : value.keys.find { |candidate| candidate.to_s == key }
|
|
715
|
+
[key, deep_sort(value.fetch(original_key))]
|
|
716
|
+
end
|
|
717
|
+
when Array
|
|
718
|
+
value.map { |item| deep_sort(item) }
|
|
719
|
+
when Pathname
|
|
720
|
+
value.to_s
|
|
721
|
+
else
|
|
722
|
+
value
|
|
723
|
+
end
|
|
724
|
+
end
|
|
725
|
+
end
|
|
726
|
+
end
|
|
727
|
+
end
|