cohere-transcribe 0.1.0

data/lib/cohere/transcribe/vad/silero.rb

@@ -0,0 +1,344 @@
+# frozen_string_literal: true
+
+require_relative "../constants"
+require_relative "../errors"
+require_relative "timestamps"
+
+module Cohere
+  module Transcribe
+    module VAD
+      class SileroBackendUnavailable < TranscriptionRuntimeError; end
+
+      # Thread-confined, bounded ONNX sequence runner for Silero VAD v6.
+      #
+      # The model consumes one row per 512-sample frame. Each row contains the
+      # preceding 64 waveform samples followed by the current frame. Recurrent
+      # h/c state and waveform context are carried across bounded temporal
+      # calls, while every new audio file starts from zero state.
+      class Silero
+        WINDOW_SAMPLES = Timestamps::WINDOW_SAMPLES
+        CONTEXT_SAMPLES = 64
+        MODEL_ROW_SAMPLES = WINDOW_SAMPLES + CONTEXT_SAMPLES
+        MAX_SEQUENCE_FRAMES = 256
+        DEFAULT_INTRA_OP_THREADS = 1
+        STATE_WIDTH = 128
+        MODEL_PATH = File.expand_path("silero_vad_v6.onnx", __dir__).freeze
+        ENGINE = "onnx"
+        PROVIDER = "CPUExecutionProvider"
+        SESSION_OPTIONS = {
+          providers: [PROVIDER].freeze,
+          inter_op_num_threads: 1,
+          intra_op_num_threads: 1,
+          enable_cpu_mem_arena: false,
+          log_severity_level: 4
+        }.freeze
+
+        Execution = Data.define(
+          :model_calls,
+          :valid_frames,
+          :padded_frames,
+          :max_files_per_call,
+          :effective_block_frames,
+          :model_load_seconds,
+          :inference_seconds,
+          :postprocess_seconds
+        ) do
+          def initialize(model_calls:, valid_frames:, padded_frames:, max_files_per_call:, effective_block_frames:,
+                         model_load_seconds: 0.0, inference_seconds: 0.0, postprocess_seconds: 0.0)
+            super
+          end
+        end
+
+        attr_reader :model_path, :block_frames, :intra_op_threads, :last_execution
+
+        def initialize(model_path: MODEL_PATH, session: nil, session_factory: nil,
+                       block_frames: MAX_SEQUENCE_FRAMES, threads: DEFAULT_INTRA_OP_THREADS)
+          @model_path = File.expand_path(model_path.to_s).freeze
+          raise ArgumentError, "block_frames must be a positive integer" unless block_frames.is_a?(Integer) && block_frames.positive?
+          raise ArgumentError, "threads must be a positive integer" unless threads.is_a?(Integer) && threads.positive?
+
+          @block_frames = block_frames
+          @intra_op_threads = threads
+          @session = session
+          @session_factory = session_factory
+          @current_model_load_seconds = 0.0
+          @current_inference_seconds = 0.0
+          @last_execution = execution(model_calls: 0, valid_frames: 0)
+        end
+
+        def engine
+          ENGINE
+        end
+
+        def provider
+          PROVIDER
+        end
+
+        # Match ONNX Runtime's execution-provider introspection. Thread counts
+        # belong to SessionOptions and are reported separately in the profile.
+        def provider_options
+          @provider_options ||= { PROVIDER => {}.freeze }.freeze
+        end
+
+        def speech_probabilities(audio)
+          @current_model_load_seconds = 0.0
+          @current_inference_seconds = 0.0
+          audio_length = validate_mono_audio!(audio)
+          if audio_length.zero?
+            @last_execution = execution(model_calls: 0, valid_frames: 0)
+            return []
+          end
+
+          return speech_probabilities_numo(audio, audio_length) if numo_array?(audio)
+
+          hidden = zero_state
+          cell = zero_state
+          previous_context = Array.new(CONTEXT_SAMPLES, 0.0)
+          frame_count = (audio_length + WINDOW_SAMPLES - 1) / WINDOW_SAMPLES
+          outputs = []
+
+          model_calls = 0
+          (0...frame_count).step(block_frames) do |frame_start|
+            chunk_frames = [block_frames, frame_count - frame_start].min
+            frames = build_frames(audio, audio_length, frame_start, chunk_frames)
+            contexts = build_contexts(frames, previous_context)
+            previous_context = frames.last.last(CONTEXT_SAMPLES).dup
+            model_rows = contexts.zip(frames).map { |context, frame| context + frame }
+
+            probabilities, hidden, cell = run_model(model_rows, hidden, cell)
+            chunk_probabilities = one_dimensional_output(probabilities)
+            unless chunk_probabilities.length == chunk_frames
+              raise TranscriptionRuntimeError,
+                    "Silero ONNX returned #{chunk_probabilities.length} probabilities " \
+                    "for #{chunk_frames} frames"
+            end
+            outputs.concat(chunk_probabilities)
+            model_calls += 1
+          end
+
+          @last_execution = execution(model_calls: model_calls, valid_frames: frame_count)
+          outputs
+        end
+
+        def speech_timestamps(
+          audio,
+          sampling_rate: SAMPLE_RATE,
+          threshold: 0.5,
+          min_speech_duration_ms: 250,
+          max_speech_duration_s: Float::INFINITY,
+          min_silence_duration_ms: 100,
+          speech_pad_ms: 30,
+          neg_threshold: nil,
+          min_silence_at_max_speech: 98,
+          use_max_poss_sil_at_max_speech: true,
+          cancel_check: nil
+        )
+          probabilities = speech_probabilities(audio)
+          started = monotonic
+          begin
+            Timestamps.from_probabilities(
+              audio.length,
+              probabilities,
+              sampling_rate: sampling_rate,
+              threshold: threshold,
+              min_speech_duration_ms: min_speech_duration_ms,
+              max_speech_duration_s: max_speech_duration_s,
+              min_silence_duration_ms: min_silence_duration_ms,
+              speech_pad_ms: speech_pad_ms,
+              neg_threshold: neg_threshold,
+              min_silence_at_max_speech: min_silence_at_max_speech,
+              use_max_poss_sil_at_max_speech: use_max_poss_sil_at_max_speech,
+              cancel_check: cancel_check
+            )
+          ensure
+            @last_execution = @last_execution.with(postprocess_seconds: monotonic - started)
+          end
+        end
+
+        private
+
+        def speech_probabilities_numo(audio, audio_length)
+          hidden = ::Numo::SFloat.zeros(1, 1, STATE_WIDTH)
+          cell = ::Numo::SFloat.zeros(1, 1, STATE_WIDTH)
+          previous_context = ::Numo::SFloat.zeros(CONTEXT_SAMPLES)
+          frame_count = (audio_length + WINDOW_SAMPLES - 1) / WINDOW_SAMPLES
+          outputs = []
+
+          model_calls = 0
+          (0...frame_count).step(block_frames) do |frame_start|
+            chunk_frames = [block_frames, frame_count - frame_start].min
+            sample_start = frame_start * WINDOW_SAMPLES
+            sample_end = [audio_length, (frame_start + chunk_frames) * WINDOW_SAMPLES].min
+            chunk_audio = audio[sample_start...sample_end].cast_to(::Numo::SFloat)
+            frames = ::Numo::SFloat.zeros(chunk_frames, WINDOW_SAMPLES)
+            full_frames, remainder = chunk_audio.size.divmod(WINDOW_SAMPLES)
+            if full_frames.positive?
+              frames[0...full_frames, true] =
+                chunk_audio[0...(full_frames * WINDOW_SAMPLES)].reshape(full_frames, WINDOW_SAMPLES)
+            end
+            frames[full_frames, 0...remainder] = chunk_audio[(full_frames * WINDOW_SAMPLES)...chunk_audio.size] if remainder.positive?
+
+            model_rows = ::Numo::SFloat.zeros(chunk_frames, MODEL_ROW_SAMPLES)
+            model_rows[0, 0...CONTEXT_SAMPLES] = previous_context
+            if chunk_frames > 1
+              model_rows[1...chunk_frames, 0...CONTEXT_SAMPLES] =
+                frames[0...(chunk_frames - 1), (WINDOW_SAMPLES - CONTEXT_SAMPLES)...WINDOW_SAMPLES]
+            end
+            model_rows[true, CONTEXT_SAMPLES...MODEL_ROW_SAMPLES] = frames
+            previous_context =
+              frames[chunk_frames - 1, (WINDOW_SAMPLES - CONTEXT_SAMPLES)...WINDOW_SAMPLES].dup
+
+            probabilities, hidden, cell = run_model(model_rows, hidden, cell, output_type: :numo)
+            chunk_probabilities = one_dimensional_output(probabilities)
+            unless chunk_probabilities.length == chunk_frames
+              raise TranscriptionRuntimeError,
+                    "Silero ONNX returned #{chunk_probabilities.length} probabilities " \
+                    "for #{chunk_frames} frames"
+            end
+            outputs.concat(chunk_probabilities)
+            model_calls += 1
+          end
+
+          @last_execution = execution(model_calls: model_calls, valid_frames: frame_count)
+          outputs
+        end
+
+        def validate_mono_audio!(audio)
+          if audio.respond_to?(:ndim) && audio.ndim != 1
+            shape = audio.respond_to?(:shape) ? audio.shape.inspect : audio.ndim.to_s
+            raise ArgumentError, "Silero VAD expects mono audio, got shape #{shape}"
+          end
+          raise ArgumentError, "Silero VAD expects an indexable mono waveform" unless audio.respond_to?(:length) && audio.respond_to?(:[])
+          raise ArgumentError, "Silero VAD expects mono audio, got nested samples" if audio.is_a?(Array) && audio.any?(Array)
+
+          length = audio.length
+          raise ArgumentError, "Silero VAD expects a non-negative audio length" unless length.is_a?(Integer) && length >= 0
+
+          length
+        end
+
+        def build_frames(audio, audio_length, frame_start, chunk_frames)
+          frames = Array.new(chunk_frames) { Array.new(WINDOW_SAMPLES, 0.0) }
+          sample_start = frame_start * WINDOW_SAMPLES
+          sample_end = [audio_length, (frame_start + chunk_frames) * WINDOW_SAMPLES].min
+          (sample_start...sample_end).each do |sample_index|
+            local_index = sample_index - sample_start
+            frame_index, within_frame = local_index.divmod(WINDOW_SAMPLES)
+            frames[frame_index][within_frame] = Float(audio[sample_index])
+          end
+          frames
+        rescue TypeError, ArgumentError => e
+          raise ArgumentError, "Silero VAD audio must contain real samples: #{e.message}"
+        end
+
+        def build_contexts(frames, previous_context)
+          contexts = Array.new(frames.length) { Array.new(CONTEXT_SAMPLES, 0.0) }
+          contexts[0] = previous_context.dup
+          1.upto(frames.length - 1) do |index|
+            contexts[index] = frames[index - 1].last(CONTEXT_SAMPLES).dup
+          end
+          contexts
+        end
+
+        def run_model(model_rows, hidden, cell, output_type: nil)
+          feed = {
+            "input" => model_rows,
+            "h" => hidden,
+            "c" => cell
+          }
+          active_session = session
+          started = monotonic
+          values = if output_type
+                     active_session.run(nil, feed, output_type: output_type)
+                   else
+                     active_session.run(nil, feed)
+                   end
+          unless values.is_a?(Array) && values.length == 3
+            raise TranscriptionRuntimeError, "Silero ONNX must return probabilities, hidden state, and cell state"
+          end
+
+          values
+        rescue StandardError => e
+          raise SileroBackendUnavailable, e.message if onnxruntime_error?(e)
+
+          raise
+        ensure
+          @current_inference_seconds += monotonic - started if started
+        end
+
+        def one_dimensional_output(values)
+          if values.respond_to?(:ndim) && values.ndim != 1
+            raise TranscriptionRuntimeError, "Silero ONNX returned a non-vector probability tensor"
+          end
+          raise TranscriptionRuntimeError, "Silero ONNX returned an invalid probability tensor" unless values.respond_to?(:to_a)
+
+          array = values.to_a
+          raise TranscriptionRuntimeError, "Silero ONNX returned a non-vector probability tensor" if array.any?(Array)
+
+          array.map { |value| Float(value) }
+        rescue TypeError, ArgumentError => e
+          raise TranscriptionRuntimeError, "Silero ONNX returned invalid probabilities: #{e.message}"
+        end
+
+        def zero_state
+          [[Array.new(STATE_WIDTH, 0.0)]]
+        end
+
+        def execution(model_calls:, valid_frames:)
+          Execution.new(
+            model_calls: model_calls,
+            valid_frames: valid_frames,
+            # The sequence graph receives one unpadded temporal stream. Only
+            # samples in the final 32 ms waveform frame can be zero-filled.
+            padded_frames: valid_frames,
+            max_files_per_call: model_calls.positive? ? 1 : 0,
+            effective_block_frames: block_frames,
+            model_load_seconds: @current_model_load_seconds,
+            inference_seconds: @current_inference_seconds,
+            postprocess_seconds: 0.0
+          )
+        end
+
+        def numo_array?(audio)
+          defined?(::Numo::NArray) && audio.is_a?(::Numo::NArray)
+        end
+
+        def session
+          return @session if @session
+
+          started = monotonic
+          @session = build_session
+        ensure
+          @current_model_load_seconds += monotonic - started if started
+        end
+
+        def build_session
+          raise SileroBackendUnavailable, "packaged Silero ONNX asset is missing: #{model_path}" unless File.file?(model_path)
+
+          return @session_factory.call(model_path, **session_options) if @session_factory
+
+          require "onnxruntime"
+          OnnxRuntime::InferenceSession.new(model_path, **session_options)
+        rescue LoadError, SystemCallError => e
+          raise SileroBackendUnavailable, "Silero ONNX backend is unavailable: #{e.message}"
+        rescue StandardError => e
+          raise SileroBackendUnavailable, e.message if onnxruntime_error?(e)
+
+          raise
+        end
+
+        def onnxruntime_error?(error)
+          error.class.name.to_s.start_with?("OnnxRuntime::")
+        end
+
+        def session_options
+          SESSION_OPTIONS.merge(intra_op_num_threads: intra_op_threads).freeze
+        end
+
+        def monotonic
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+      end
+    end
+  end
+end

data/lib/cohere/transcribe/vad/timestamps.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+module Cohere
+  module Transcribe
+    module VAD
+      # Silero 6.2.1's sample-domain timestamp state machine.
+      #
+      # Keeping this separate from the inference backend makes every Silero
+      # engine share exactly the same threshold, hysteresis, duration, and
+      # padding behavior.
+      module Timestamps
+        WINDOW_SAMPLES = 512
+        CANCELLATION_CHECK_FRAMES = 4_096
+
+        module_function
+
+        def from_probabilities(
+          audio_length_samples,
+          speech_probabilities,
+          sampling_rate: 16_000,
+          threshold: 0.5,
+          min_speech_duration_ms: 250,
+          max_speech_duration_s: Float::INFINITY,
+          min_silence_duration_ms: 100,
+          speech_pad_ms: 30,
+          neg_threshold: nil,
+          min_silence_at_max_speech: 98,
+          use_max_poss_sil_at_max_speech: true,
+          cancel_check: nil
+        )
+          check_cancellation(cancel_check)
+          raise ArgumentError, "The vectorized Silero v6 export requires 16 kHz audio" unless sampling_rate == 16_000
+          unless audio_length_samples.is_a?(Integer) && audio_length_samples >= 0
+            raise ArgumentError, "audio_length_samples must be a non-negative integer"
+          end
+
+          speech_probs = one_dimensional_float32(speech_probabilities)
+          expected_frames = (audio_length_samples + WINDOW_SAMPLES - 1) / WINDOW_SAMPLES
+          unless speech_probs.length == expected_frames
+            raise ArgumentError,
+                  "Silero probability count does not match the audio length: " \
+                  "expected #{expected_frames}, got #{speech_probs.length}"
+          end
+          raise ArgumentError, "Silero probabilities contain non-finite values" unless speech_probs.all?(&:finite?)
+          unless speech_probs.all? { |probability| probability.between?(0.0, 1.0) }
+            raise ArgumentError, "Silero probabilities must be between zero and one"
+          end
+
+          check_cancellation(cancel_check)
+
+          min_speech_samples = sampling_rate * min_speech_duration_ms / 1000.0
+          speech_pad_samples = sampling_rate * speech_pad_ms / 1000.0
+          max_speech_samples = (
+            (sampling_rate * max_speech_duration_s) - WINDOW_SAMPLES - (2 * speech_pad_samples)
+          )
+          min_silence_samples = sampling_rate * min_silence_duration_ms / 1000.0
+          min_silence_samples_at_max_speech = sampling_rate * min_silence_at_max_speech / 1000.0
+          # NumPy keeps scalar comparisons in the probability array's float32
+          # dtype. Coerce the comparison thresholds likewise so decimal values
+          # such as 0.01 and 0.35 do not move a frame across a boundary merely
+          # because Ruby Float is binary64.
+          positive_threshold = float32(threshold)
+          negative_threshold = float32(neg_threshold || [threshold - 0.15, 0.01].max)
+
+          triggered = false
+          speeches = []
+          current_speech = {}
+          temp_end = 0
+          prev_end = 0
+          next_start = 0
+          possible_ends = []
+          next_cancellation_check = CANCELLATION_CHECK_FRAMES
+
+          speech_probs.each_with_index do |speech_prob, index|
+            if index == next_cancellation_check
+              check_cancellation(cancel_check)
+              next_cancellation_check += CANCELLATION_CHECK_FRAMES
+            end
+            current_sample = WINDOW_SAMPLES * index
+
+            if speech_prob >= positive_threshold && temp_end.positive?
+              silence_duration = current_sample - temp_end
+              possible_ends << [temp_end, silence_duration] if silence_duration > min_silence_samples_at_max_speech
+              temp_end = 0
+              next_start = current_sample if next_start < prev_end
+            end
+
+            if speech_prob >= positive_threshold && !triggered
+              triggered = true
+              current_speech[:start] = current_sample
+              next
+            end
+
+            if triggered && current_sample - current_speech.fetch(:start) > max_speech_samples
+              if use_max_poss_sil_at_max_speech && !possible_ends.empty?
+                prev_end, silence_duration = possible_ends.max_by { |_end_sample, duration| duration }
+                current_speech[:end] = prev_end
+                speeches << current_speech
+                current_speech = {}
+                next_start = prev_end + silence_duration
+                if next_start < prev_end + current_sample
+                  current_speech[:start] = next_start
+                else
+                  triggered = false
+                end
+                prev_end = next_start = temp_end = 0
+                possible_ends = []
+              elsif prev_end.positive?
+                current_speech[:end] = prev_end
+                speeches << current_speech
+                current_speech = {}
+                if next_start < prev_end
+                  triggered = false
+                else
+                  current_speech[:start] = next_start
+                end
+                prev_end = next_start = temp_end = 0
+                possible_ends = []
+              else
+                current_speech[:end] = current_sample
+                speeches << current_speech
+                current_speech = {}
+                prev_end = next_start = temp_end = 0
+                triggered = false
+                possible_ends = []
+                next
+              end
+            end
+
+            next unless speech_prob < negative_threshold && triggered
+
+            temp_end = current_sample unless temp_end.positive?
+            current_silence_duration = current_sample - temp_end
+            if !use_max_poss_sil_at_max_speech &&
+               current_silence_duration > min_silence_samples_at_max_speech
+              prev_end = temp_end
+            end
+            next if current_silence_duration < min_silence_samples
+
+            current_speech[:end] = temp_end
+            speeches << current_speech if current_speech.fetch(:end) - current_speech.fetch(:start) > min_speech_samples
+            current_speech = {}
+            prev_end = next_start = temp_end = 0
+            triggered = false
+            possible_ends = []
+          end
+
+          check_cancellation(cancel_check)
+          if !current_speech.empty? &&
+             audio_length_samples - current_speech.fetch(:start) > min_speech_samples
+            current_speech[:end] = audio_length_samples
+            speeches << current_speech
+          end
+
+          pad_speeches!(speeches, audio_length_samples, speech_pad_samples, cancel_check)
+          check_cancellation(cancel_check)
+          speeches
+        end
+
+        def one_dimensional_float32(values)
+          if values.respond_to?(:ndim) && values.ndim != 1
+            shape = values.respond_to?(:shape) ? values.shape.inspect : values.ndim.to_s
+            raise ArgumentError, "Silero probabilities must be one-dimensional, got #{shape}"
+          end
+          raise ArgumentError, "Silero probabilities must be one-dimensional" unless values.respond_to?(:to_a)
+
+          array = values.to_a
+          if array.any? { |value| value.is_a?(Array) || value.respond_to?(:ndim) }
+            raise ArgumentError, "Silero probabilities must be one-dimensional"
+          end
+
+          array.map { |value| float32(value) }
+        rescue TypeError, ArgumentError => e
+          raise e if e.message.start_with?("Silero probabilities")
+
+          raise ArgumentError, "Silero probabilities must contain real numbers: #{e.message}"
+        end
+        private_class_method :one_dimensional_float32
+
+        def float32(value)
+          [Float(value)].pack("f").unpack1("f")
+        end
+        private_class_method :float32
+
+        def pad_speeches!(speeches, audio_length_samples, speech_pad_samples, cancel_check)
+          next_cancellation_check = CANCELLATION_CHECK_FRAMES
+          speeches.each_with_index do |speech, index|
+            if index == next_cancellation_check
+              check_cancellation(cancel_check)
+              next_cancellation_check += CANCELLATION_CHECK_FRAMES
+            end
+
+            speech[:start] = [0, speech.fetch(:start) - speech_pad_samples].max.to_i if index.zero?
+            if index == speeches.length - 1
+              speech[:end] = [audio_length_samples, speech.fetch(:end) + speech_pad_samples].min.to_i
+            else
+              following = speeches[index + 1]
+              silence_duration = following.fetch(:start) - speech.fetch(:end)
+              if silence_duration < 2 * speech_pad_samples
+                half_silence = silence_duration.div(2)
+                speech[:end] += half_silence
+                following[:start] = [0, following.fetch(:start) - half_silence].max.to_i
+              else
+                speech[:end] = [audio_length_samples, speech.fetch(:end) + speech_pad_samples].min.to_i
+                following[:start] = [0, following.fetch(:start) - speech_pad_samples].max.to_i
+              end
+            end
+          end
+        end
+        private_class_method :pad_speeches!
+
+        def check_cancellation(cancel_check)
+          cancel_check&.call
+        end
+        private_class_method :check_cancellation
+      end
+    end
+  end
+end

data/lib/cohere/transcribe/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Cohere
+  module Transcribe
+    VERSION = "0.1.0"
+  end
+end

data/lib/cohere/transcribe.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative "transcribe/version"
+require_relative "transcribe/constants"
+require_relative "transcribe/errors"
+require_relative "transcribe/types"
+require_relative "transcribe/input"
+require_relative "transcribe/loader"
+require_relative "transcribe/api"
+
+module Cohere
+  module Transcribe
+    # Ruby counterpart to the Python package's explicit root export contract.
+    # Internal implementation constants remain reachable through autoloads but
+    # are intentionally absent from this stable list.
+    PUBLIC_API = %w[
+      BatchTranscriptionError Error ProgressCallbackError ProgressEvent
+      PublicationOptions SubtitleCue Transcriber TranscriberBusyError
+      TranscriberClosedError TranscriptionConfigurationError TranscriptionError
+      TranscriptionInputError TranscriptionOptions TranscriptionProvenance
+      TranscriptionResult TranscriptionRun TranscriptionRuntimeError
+      TranscriptionSegment TranscriptionStatistics TranscriptionWord VERSION
+      transcribe
+    ].freeze
+  end
+end