ruby_llm 1.16.0 → 2.0.0.rc2

data/lib/ruby_llm/transcription/wav_audio.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class Transcription
+    class WavAudio # :nodoc: all
+      attr_reader :data, :sample_rate, :channels, :bits_per_sample, :encoding
+
+      def initialize(content)
+        content = wav_content(content)
+        parse_chunks(content)
+        raise ArgumentError, 'WAV file must contain audio data and format information' unless @data && @sample_rate
+      end
+
+      def duration
+        data.bytesize.fdiv(sample_rate * channels * bits_per_sample / 8)
+      end
+
+      private
+
+      def wav_content(content)
+        unless content.start_with?('RIFF') && content.byteslice(8, 4) == 'WAVE'
+          raise ArgumentError, 'This streaming transcription endpoint requires a WAV file'
+        end
+
+        declared_size = content.byteslice(4, 4).unpack1('V')
+        return content if declared_size == 0xFFFFFFFF
+        raise ArgumentError, 'WAV file is truncated' if content.bytesize < declared_size + 8
+
+        content.byteslice(0, declared_size + 8)
+      end
+
+      def parse_chunks(content)
+        offset = 12
+        while offset + 8 <= content.bytesize
+          name = content.byteslice(offset, 4)
+          length = content.byteslice(offset + 4, 4).unpack1('V')
+          length = content.bytesize - offset - 8 if name == 'data' && length == 0xFFFFFFFF
+          body = content.byteslice(offset + 8, length)
+          validate_chunk_length(body, length)
+
+          parse_format(body) if name == 'fmt '
+          @data = body.b if name == 'data'
+          offset += 8 + length + (length % 2)
+        end
+        raise ArgumentError, 'WAV file contains a truncated chunk or padding' unless offset == content.bytesize
+      end
+
+      def validate_chunk_length(body, length)
+        raise ArgumentError, 'WAV file contains a truncated chunk' unless body && body.bytesize == length
+      end
+
+      def parse_format(data)
+        raise ArgumentError, 'WAV file contains an invalid audio format' if data.bytesize < 16
+
+        @encoding, @channels, @sample_rate, _, _, @bits_per_sample = data.unpack('vvVVvv')
+        return if @channels.positive? && @sample_rate.positive? && @bits_per_sample.positive?
+
+        raise ArgumentError, 'WAV file contains an invalid audio format'
+      end
+    end
+  end
+end

data/lib/ruby_llm/transcription.rb

@@ -1,11 +1,41 @@
 # frozen_string_literal: true
 
 module RubyLLM
-  # Represents a transcription of audio content.
+  # A Transcription is text produced from spoken audio. RubyLLM.transcribe
+  # returns one. It holds the transcript along with any metadata the provider
+  # reports, such as language, duration, and timed segments.
+  #
+  #   transcription = RubyLLM.transcribe("meeting.wav")
+  #   transcription.text   # => "Welcome to today's meeting..."
+  #   transcription.model  # => "gpt-transcribe"
+  #
   class Transcription
-    attr_reader :text, :model, :language, :duration, :segments, :words, :input_tokens, :output_tokens
+    include Support::Inspectable
+    include Accounting::Usage::Result
 
-    def initialize(text:, model:, **attributes)
+    # The transcribed text.
+    attr_reader :text
+
+    # The id of the model that produced the transcription.
+    attr_reader :model
+
+    # The language of the audio, or +nil+ when the provider does not report it.
+    attr_reader :language
+
+    # The audio duration in seconds, or +nil+ when the provider does not
+    # report it.
+    attr_reader :duration
+
+    # The timed segments of the transcript as an array of hashes, or +nil+
+    # when the provider does not return segments. Diarization models add a
+    # speaker label to each segment.
+    attr_reader :segments
+
+    # Word timing and speaker labels as an array of hashes, or +nil+ when
+    # the provider does not return them. Request timing with +timestamps:+.
+    attr_reader :words
+
+    def initialize(text:, model:, **attributes) # :nodoc:
       @text = text
       @model = model
       @language = attributes[:language]
@@ -14,23 +44,118 @@ module RubyLLM
       @words = attributes[:words]
       @input_tokens = attributes[:input_tokens]
       @output_tokens = attributes[:output_tokens]
+      @reported_cost = attributes[:reported_cost]
     end
 
-    def self.transcribe(audio_file, **kwargs)
-      model = kwargs.delete(:model)
-      language = kwargs.delete(:language)
-      provider = kwargs.delete(:provider)
-      assume_model_exists = kwargs.delete(:assume_model_exists) { false }
-      context = kwargs.delete(:context)
-      options = kwargs
+    # Returns usage aggregated across every provider attempt.
+    def tokens
+      return ruby_llm_usage_tokens unless ruby_llm_usage_entries.empty?
+
+      Tokens.new(input: @input_tokens, output: @output_tokens, reported_cost: @reported_cost)
+    end
 
+    # Returns the transcription cost across every provider attempt.
+    def cost
+      return ruby_llm_usage_cost unless ruby_llm_usage_entries.empty?
+
+      Cost.new(tokens:, model: model_info, category: :audio_tokens)
+    end
+
+    def model_info # :nodoc:
+      @model_info ||= RubyLLM.models.find(model)
+    rescue ModelNotFoundError
+      nil
+    end
+
+    # Transcribes +audio_file+ and returns a Transcription. The file may be
+    # a path, URL, or IO object. Uses
+    # <tt>config.default_transcription_model</tt> unless +model:+ is given.
+    # Pass +provider:+ and <tt>assume_model_exists: true</tt> to use a model
+    # that is not in the registry.
+    #
+    #   RubyLLM.transcribe("meeting.wav")
+    #   RubyLLM.transcribe("entrevista.mp3", language: "es")
+    #   RubyLLM.transcribe(
+    #     "team-meeting.wav",
+    #     model: "gpt-4o-transcribe-diarize",
+    #     speaker_names: ["Alice", "Bob"],
+    #     speaker_references: ["alice-voice.wav", "bob-voice.wav"]
+    #   )
+    #
+    # +language:+ hints at the spoken language using the provider's accepted
+    # ISO 639-1 or BCP-47 language code.
+    # +prompt:+ gives the model vocabulary or formatting guidance, and
+    # +temperature:+ adjusts sampling. +format:+ selects the transcript
+    # format in the provider's own vocabulary: OpenAI takes values such as
+    # <tt>"text"</tt>, <tt>"verbose_json"</tt>, or <tt>"diarized_json"</tt>,
+    # while Gemini takes a MIME type such as <tt>"text/plain"</tt>.
+    # +speaker_names:+ and +speaker_references:+ label the speakers on
+    # models that support diarization; references may be paths, URLs, or IO
+    # objects. Option support depends on the selected model and provider.
+    # +timestamps:+ requests +:word+ timestamps. Some providers also accept
+    # +:segment+ or +:character+; unsupported granularities raise ArgumentError.
+    # +provider_options:+ takes options in the provider's request vocabulary
+    # and merges them into the rendered request as-is.
+    #
+    # Given a block, the transcript streams: each TranscriptionChunk is
+    # yielded as it arrives and the completed Transcription is still
+    # returned. Partial chunks replace earlier tentative text; only
+    # +delta+ fields append to the committed transcript. WebSocket-based
+    # providers require the optional +websocket-driver+ gem.
+    #
+    #   transcription = RubyLLM.transcribe("meeting.wav", model: "gpt-4o-transcribe") do |chunk|
+    #     print chunk.delta
+    #   end
+    #
+    # Raises RubyLLM::ModelNotFoundError if +model:+ is not in the registry,
+    # and RubyLLM::Error when a block is given to a provider that does not
+    # stream transcriptions.
+    def self.transcribe(audio_file,
+                        model: nil,
+                        language: nil,
+                        provider: nil,
+                        assume_model_exists: false,
+                        context: nil,
+                        prompt: nil,
+                        temperature: nil,
+                        format: nil,
+                        timestamps: nil,
+                        speaker_names: nil,
+                        speaker_references: nil,
+                        provider_options: {},
+                        metadata: nil,
+                        &block)
       config = context&.config || RubyLLM.config
       model ||= config.default_transcription_model
-      model, provider_instance = Models.resolve(model, provider: provider, assume_exists: assume_model_exists,
+      model, provider_instance = Models.resolve(model, provider: provider, assume_model_exists: assume_model_exists,
                                                        config: config)
-      model_id = model.id
+      empty_tokens = Tokens.new
+      payload = {
+        provider: provider_instance.slug,
+        provider_class: provider_instance.class.display_name,
+        model: model.id,
+        model_info: model,
+        language: language,
+        provider_options: provider_options,
+        metadata: metadata,
+        tokens: empty_tokens,
+        cost: Cost.new(tokens: empty_tokens, model:, category: :audio_tokens)
+      }
+
+      RubyLLM.instrument('transcription.ruby_llm', payload, config: config) do |event|
+        result = provider_instance.transcribe(audio_file, model:, language:, format:, timestamps:, speaker_names:,
+                                                          speaker_references:, provider_options:, prompt:,
+                                                          temperature:, &block)
+        event[:result] = result
+        event[:response_model] = result.model
+        event[:tokens] = result.tokens
+        event[:cost] = result.cost
+        result
+      end
+    end
 
-      provider_instance.transcribe(audio_file, model: model_id, language:, **options)
+    def inspect_attributes # :nodoc:
+      { text: text, model: model, language: language, duration: duration }
     end
   end
 end

data/lib/ruby_llm/transcription_chunk.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  # A TranscriptionChunk is one event from a streaming transcription.
+  # RubyLLM.transcribe yields these to its block as the provider transcribes
+  # the audio, then returns the final Transcription.
+  #
+  #   RubyLLM.transcribe("meeting.wav", model: "gpt-4o-transcribe") do |chunk|
+  #     print chunk.delta if chunk.delta?
+  #   end
+  #
+  class TranscriptionChunk
+    include Support::Inspectable
+
+    # Text deltas, arriving as the model transcribes.
+    DELTA = 'transcript.text.delta'
+
+    # A tentative transcript that may change before its segment completes.
+    PARTIAL = 'transcript.text.partial'
+
+    # A completed segment, on models that return timed or diarized segments.
+    SEGMENT = 'transcript.text.segment'
+
+    # The final event, carrying the complete transcript.
+    DONE = 'transcript.text.done'
+
+    # The normalized event type, such as
+    # <tt>"transcript.text.delta"</tt>.
+    attr_reader :type
+
+    # The text added by this event, or +nil+ for events that add no text.
+    attr_reader :delta
+
+    # The transcript for a partial event or the complete final transcript.
+    attr_reader :text
+
+    # The segment this event completed as a Hash, or +nil+. Diarization
+    # models label each segment with a speaker.
+    attr_reader :segment
+
+    # The parsed provider event, for fields RubyLLM does not normalize.
+    attr_reader :raw
+
+    def initialize(type:, delta: nil, text: nil, segment: nil, raw: nil) # :nodoc:
+      @type = type
+      @delta = delta
+      @text = text
+      @segment = segment
+      @raw = raw
+    end
+
+    # Whether this event carries a text delta.
+    def delta? = !delta.nil?
+
+    # Whether this is a tentative transcript, replacing the previous partial.
+    def partial? = type == PARTIAL
+
+    # Whether this event completed a segment.
+    def segment? = !segment.nil?
+
+    # Whether this is the final event of the transcription.
+    def done? = type == DONE
+
+    def inspect_attributes # :nodoc:
+      { type: type, delta: delta, text: text, segment: segment }
+    end
+  end
+end

data/lib/ruby_llm/transport/connection.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'faraday/multipart'
+require 'faraday/retry'
+require 'ruby_llm/transport/error_middleware'
+require 'ruby_llm/transport/usage_middleware'
+require 'timeout'
+
+module RubyLLM
+  module Transport # :nodoc:
+    class Connection # :nodoc:
+      include Support::Inspectable
+
+      IDEMPOTENT_KEY = :ruby_llm_idempotent
+      STREAM_PROGRESS_KEY = :ruby_llm_stream_progress
+
+      attr_reader :provider, :connection, :config
+
+      def self.basic(config = RubyLLM.config, &)
+        Faraday.new do |f|
+          f.options.timeout = config.request_timeout
+          f.proxy = config.http_proxy if config.http_proxy
+          f.response :logger,
+                     RubyLLM.logger,
+                     bodies: false,
+                     errors: true,
+                     headers: false,
+                     log_level: :debug
+          f.response :raise_error
+          yield f if block_given?
+        end
+      end
+
+      def initialize(provider, config, api_base: nil)
+        @provider = provider
+        @config = config
+
+        @connection = Faraday.new(api_base || provider.api_base) do |faraday|
+          setup_timeout(faraday)
+          setup_logging(faraday)
+          setup_retry(faraday)
+          setup_middleware(faraday)
+          setup_http_proxy(faraday)
+        end
+      end
+
+      def post(url, payload, usage: nil, idempotent: true, &)
+        instrument_request(:post, url) do
+          @connection.post url, payload do |req|
+            req.headers.merge! @provider.headers
+            set_usage_tracker(req, usage) if usage
+            mark_non_idempotent(req) unless idempotent
+            yield req if block_given?
+          end
+        end
+      end
+
+      def get(url, &)
+        instrument_request(:get, url) do
+          @connection.get url do |req|
+            req.headers.merge! @provider.headers
+            yield req if block_given?
+          end
+        end
+      end
+
+      def patch(url, payload, &)
+        instrument_request(:patch, url) do
+          @connection.patch url, payload do |req|
+            req.headers.merge! @provider.headers
+            yield req if block_given?
+          end
+        end
+      end
+
+      def delete(url, &)
+        instrument_request(:delete, url) do
+          @connection.delete url do |req|
+            req.headers.merge! @provider.headers
+            yield req if block_given?
+          end
+        end
+      end
+
+      private
+
+      def instrument_request(method, url)
+        payload = {
+          provider: @provider.slug,
+          method: method,
+          url: url
+        }
+
+        RubyLLM.instrument('request.ruby_llm', payload, config: @config) do |event|
+          response = yield
+          event[:status] = response.status if response.respond_to?(:status)
+          response
+        end
+      end
+
+      def setup_timeout(faraday)
+        faraday.options.timeout = @config.request_timeout
+      end
+
+      def setup_logging(faraday)
+        faraday.response :logger,
+                         RubyLLM.logger,
+                         bodies: RubyLLM.logger.debug?,
+                         errors: true,
+                         headers: false,
+                         log_level: :debug do |logger|
+          logger.filter(logging_regexp('[A-Za-z0-9+/=]{100,}'), '[BASE64 DATA]')
+          logger.filter(logging_regexp('[-\\d.e,\\s]{100,}'), '[EMBEDDINGS ARRAY]')
+        end
+      end
+
+      def logging_regexp(pattern)
+        return Regexp.new(pattern) if @config.log_regexp_timeout.nil? || !Regexp.respond_to?(:timeout)
+
+        Regexp.new(pattern, timeout: @config.log_regexp_timeout)
+      end
+
+      def setup_retry(faraday)
+        faraday.request :retry, {
+          max: @config.max_retries,
+          interval: @config.retry_interval,
+          max_interval: @config.retry_max_interval,
+          interval_randomness: @config.retry_interval_randomness,
+          backoff_factor: @config.retry_backoff_factor,
+          methods: Faraday::Retry::Middleware::IDEMPOTENT_METHODS,
+          retry_if: lambda { |env, _exception|
+            env[:method] == :post && idempotent?(env) && !stream_delivered?(env)
+          },
+          exceptions: retry_exceptions
+        }
+        faraday.use :llm_usage
+      end
+
+      def stream_delivered?(env)
+        env[:request]&.context&.dig(STREAM_PROGRESS_KEY, :started)
+      end
+
+      def idempotent?(env)
+        env[:request]&.context&.dig(IDEMPOTENT_KEY) != false
+      end
+
+      def setup_middleware(faraday)
+        faraday.request :multipart
+        faraday.request :json
+        faraday.response :json
+        faraday.adapter(@config.faraday_adapter)
+        faraday.use :llm_errors, provider: @provider
+      end
+
+      def setup_http_proxy(faraday)
+        return unless @config.http_proxy
+
+        faraday.proxy = @config.http_proxy
+      end
+
+      def retry_exceptions
+        [
+          Errno::ETIMEDOUT,
+          Timeout::Error,
+          Faraday::TimeoutError,
+          Faraday::ConnectionFailed,
+          Faraday::RetriableResponse,
+          RubyLLM::RateLimitError,
+          RubyLLM::ServerError,
+          RubyLLM::ServiceUnavailableError,
+          RubyLLM::OverloadedError
+        ]
+      end
+
+      def set_usage_tracker(request, tracker)
+        context = request.options.context ||= {}
+        context[UsageMiddleware::CONTEXT_KEY] = tracker
+      end
+
+      # A request that creates server-side state cannot be replayed: a retry
+      # after a lost response submits the job a second time.
+      def mark_non_idempotent(request)
+        context = request.options.context ||= {}
+        context[IDEMPOTENT_KEY] = false
+      end
+
+      def inspect_attributes # :nodoc:
+        { provider: @provider.slug }
+      end
+    end
+  end
+end

data/lib/ruby_llm/transport/error_middleware.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'ruby_llm/error'
+
+module RubyLLM
+  module Transport # :nodoc:
+    class ErrorMiddleware < Faraday::Middleware # :nodoc: all
+      def initialize(app, options = {})
+        super(app)
+        @provider = options[:provider]
+      end
+
+      # Sits directly above the adapter, inside the retry middleware, so this
+      # runs once per attempt: streaming state stored on the env by a previous
+      # attempt must not leak into the next one.
+      def call(env)
+        env[:streaming_error_response] = nil
+        env[:streaming_state] = nil
+        @app.call(env).on_complete do |response|
+          apply_retry_delay(response)
+          self.class.parse_error(provider: @provider, response: streaming_error_response(response))
+        end
+      end
+
+      private
+
+      # The retry middleware only reads the standard Retry-After header, so
+      # provider-specific rate-limit headers are normalized into it here,
+      # where the provider is known.
+      def apply_retry_delay(response)
+        status = response.respond_to?(:status) ? response.status : response[:status]
+        return unless status == 429
+
+        headers = response[:response_headers]
+        if @provider && !headers['Retry-After'] && (delay = @provider.retry_delay(response))
+          headers['Retry-After'] = delay.to_s
+        end
+      end
+
+      def streaming_error_response(response)
+        stored_response = if response.respond_to?(:env) && response.env.respond_to?(:[])
+                            response.env[:streaming_error_response]
+                          elsif response.respond_to?(:[])
+                            response[:streaming_error_response]
+                          end
+
+        stored_response || response
+      rescue NameError
+        response
+      end
+
+      class << self
+        CONTEXT_LENGTH_PATTERNS = [
+          /context length/i,
+          /context window/i,
+          /exceeds?.*context size/i,
+          /maximum context/i,
+          /request too large/i,
+          /too many tokens/i,
+          /token count exceeds/i,
+          /input[_\s-]?token/i,
+          /input or output tokens? must be reduced/i,
+          /reduce the length of messages/i,
+          /prompt is too long/i,
+          /context limit/i
+        ].freeze
+
+        RATE_LIMIT_PATTERNS = [
+          /rate limit/i,
+          /per minute/i,
+          /per hour/i,
+          /per day/i
+        ].freeze
+
+        def parse_error(provider:, response:)
+          message = provider&.parse_error(response)
+
+          case response.status
+          when 200..399
+            message
+          when 400
+            raise ContextLengthExceededError.new(message, response:) if context_length_exceeded?(message)
+
+            raise BadRequestError.new(message, response:)
+          when 401
+            raise UnauthorizedError.new(message, response:)
+          when 402
+            raise PaymentRequiredError.new(message, response:)
+          when 403
+            raise ForbiddenError.new(message, response:)
+          when 429
+            raise RateLimitError.new(message, response:) if rate_limited?(message)
+            raise ContextLengthExceededError.new(message, response:) if context_length_exceeded?(message)
+
+            raise RateLimitError.new(message, response:)
+          when 500
+            raise ServerError.new(message, response:)
+          when 502..504
+            raise ServiceUnavailableError.new(message, response:)
+          when 529
+            raise OverloadedError.new(message, response:)
+          else
+            raise Error.new(message, response:)
+          end
+        end
+
+        private
+
+        # Providers hand back whatever their error body holds, which is not
+        # always a String: bedrock-mantle nests code, message, and type in a
+        # Hash. Match on the rendered text so any shape classifies.
+        def context_length_exceeded?(message)
+          text = message.to_s
+          return false if text.empty?
+
+          CONTEXT_LENGTH_PATTERNS.any? { |pattern| text.match?(pattern) }
+        end
+
+        def rate_limited?(message)
+          text = message.to_s
+          return false if text.empty?
+
+          RATE_LIMIT_PATTERNS.any? { |pattern| text.match?(pattern) }
+        end
+      end
+    end
+  end
+end
+
+Faraday::Middleware.register_middleware(llm_errors: RubyLLM::Transport::ErrorMiddleware)

data/lib/ruby_llm/transport/usage_middleware.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'faraday'
+
+module RubyLLM
+  module Transport # :nodoc:
+    # Sits inside Faraday retry middleware so every transport attempt produces
+    # one usage observation.
+    class UsageMiddleware < Faraday::Middleware # :nodoc: all
+      CONTEXT_KEY = :ruby_llm_usage_tracker
+
+      def call(env)
+        tracker = env.request.context&.[](CONTEXT_KEY)
+        return @app.call(env) unless tracker
+
+        entry = tracker.start
+        begin
+          @app.call(env)
+        rescue StandardError => e
+          tracker.fail_attempt(entry, e)
+          raise
+        end
+      end
+    end
+  end
+end
+
+Faraday::Middleware.register_middleware(llm_usage: RubyLLM::Transport::UsageMiddleware)