ruby_llm 2.0.0 → 2.1.0

data/lib/ruby_llm/transport/error_middleware.rb

@@ -6,6 +6,9 @@ require 'ruby_llm/error'
 module RubyLLM
   module Transport # :nodoc:
     class ErrorMiddleware < Faraday::Middleware # :nodoc: all
+      PROVIDER_KEY = :ruby_llm_provider
+      STREAM_RESET_KEY = :ruby_llm_stream_reset
+
       def initialize(app, options = {})
         super(app)
         @provider = options[:provider]
@@ -17,25 +20,37 @@ module RubyLLM
       def call(env)
         env[:streaming_error_response] = nil
         env[:streaming_state] = nil
+        context = env[:request]&.context
+        context&.[](STREAM_RESET_KEY)&.call
+        provider = context&.[](PROVIDER_KEY) || @provider
         @app.call(env).on_complete do |response|
-          apply_retry_delay(response)
-          self.class.parse_error(provider: @provider, response: streaming_error_response(response))
+          apply_retry_delay(response, provider)
+          self.class.parse_error(provider:, response: streaming_error_response(response))
         end
       end
 
       private
 
       # The retry middleware only reads the standard Retry-After header, so
-      # provider-specific rate-limit headers are normalized into it here,
-      # where the provider is known.
-      def apply_retry_delay(response)
+      # other retry hints are normalized into seconds here.
+      def apply_retry_delay(response, provider)
         status = response.respond_to?(:status) ? response.status : response[:status]
-        return unless status == 429
+        return unless status && status >= 400
 
         headers = response[:response_headers]
-        if @provider && !headers['Retry-After'] && (delay = @provider.retry_delay(response))
-          headers['Retry-After'] = delay.to_s
-        end
+        return unless headers && !headers['Retry-After']
+
+        delay = millisecond_retry_delay(headers) || provider_retry_delay(provider, response, status)
+        headers['Retry-After'] = delay.to_s if delay
+      end
+
+      def provider_retry_delay(provider, response, status)
+        provider&.retry_delay(response) if status == 429
+      end
+
+      def millisecond_retry_delay(headers)
+        value = Float(headers['retry-after-ms'], exception: false)
+        value / 1000 if value&.finite? && value >= 0
       end
 
       def streaming_error_response(response)
@@ -56,7 +71,7 @@ module RubyLLM
           /context window/i,
           /exceeds?.*context size/i,
           /maximum context/i,
-          /request too large/i,
+          /request (?:was )?too large/i,
           /too many tokens/i,
           /token count exceeds/i,
           /input[_\s-]?token/i,
@@ -70,25 +85,41 @@ module RubyLLM
           /rate limit/i,
           /per minute/i,
           /per hour/i,
-          /per day/i
+          /per day/i,
+          /quota exceeded/i,
+          /wait before trying again/i
+        ].freeze
+
+        OVERLOAD_PATTERNS = [
+          /currently overloaded/i
+        ].freeze
+
+        PAYMENT_REQUIRED_PATTERNS = [
+          /credit balance is too low/i
+        ].freeze
+
+        AUTHENTICATION_PATTERNS = [
+          /api key not valid/i,
+          /incorrect api key/i,
+          /security token included in the request is (?:invalid|expired)/i
         ].freeze
 
         def parse_error(provider:, response:)
+          return if (200..399).cover?(response.status)
+
           message = provider&.parse_error(response)
 
           case response.status
-          when 200..399
-            message
           when 400
-            raise ContextLengthExceededError.new(message, response:) if context_length_exceeded?(message)
-
-            raise BadRequestError.new(message, response:)
+            raise_bad_request(message, response)
           when 401
             raise UnauthorizedError.new(message, response:)
           when 402
             raise PaymentRequiredError.new(message, response:)
           when 403
-            raise ForbiddenError.new(message, response:)
+            raise_forbidden(message, response)
+          when 413
+            raise_too_large(message, response)
           when 429
             raise RateLimitError.new(message, response:) if rate_limited?(message)
             raise ContextLengthExceededError.new(message, response:) if context_length_exceeded?(message)
@@ -107,21 +138,56 @@ module RubyLLM
 
         private
 
-        # Providers hand back whatever their error body holds, which is not
-        # always a String: bedrock-mantle nests code, message, and type in a
-        # Hash. Match on the rendered text so any shape classifies.
+        def raise_bad_request(message, response)
+          raise UnauthorizedError.new(message, response:) if authentication_failed?(message)
+          raise ContextLengthExceededError.new(message, response:) if context_length_exceeded?(message)
+          raise RateLimitError.new(message, response:) if rate_limited?(message)
+          raise OverloadedError.new(message, response:) if overloaded?(message)
+          raise PaymentRequiredError.new(message, response:) if payment_required?(message)
+
+          raise BadRequestError.new(message, response:)
+        end
+
+        def raise_forbidden(message, response)
+          raise UnauthorizedError.new(message, response:) if authentication_failed?(message)
+
+          raise ForbiddenError.new(message, response:)
+        end
+
+        def raise_too_large(message, response)
+          raise ContextLengthExceededError.new(message, response:) if context_length_exceeded?(message)
+
+          raise Error.new(message, response:)
+        end
+
         def context_length_exceeded?(message)
-          text = message.to_s
-          return false if text.empty?
+          matches?(message, CONTEXT_LENGTH_PATTERNS)
+        end
+
+        def overloaded?(message)
+          matches?(message, OVERLOAD_PATTERNS)
+        end
 
-          CONTEXT_LENGTH_PATTERNS.any? { |pattern| text.match?(pattern) }
+        def payment_required?(message)
+          matches?(message, PAYMENT_REQUIRED_PATTERNS)
         end
 
         def rate_limited?(message)
+          matches?(message, RATE_LIMIT_PATTERNS)
+        end
+
+        def authentication_failed?(message)
+          matches?(message, AUTHENTICATION_PATTERNS)
+        end
+
+        # Providers hand back whatever their error body holds, which is not
+        # always a String: bedrock-mantle nests code, message, and type in a
+        # Hash. Match on the rendered text so any shape classifies.
+        def matches?(message, patterns)
           text = message.to_s
           return false if text.empty?
 
-          RATE_LIMIT_PATTERNS.any? { |pattern| text.match?(pattern) }
+          patterns.any? { |pattern| text.match?(pattern) }
         end
       end
     end

data/lib/ruby_llm/transport/event_stream_parser.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Transport
+    # Interprets a text/event-stream as the HTML standard describes, fed in
+    # whatever pieces the network delivers. Each dispatched event is yielded
+    # as its type, data, and last event ID. The last event ID and the
+    # reconnection time the server asked for stay readable between events.
+    class EventStreamParser # :nodoc:
+      BOM = "\xEF\xBB\xBF".b.freeze
+      LINE_BREAK = /[\r\n]/
+      DIGITS = /\A[0-9]+\z/
+      CR = 13
+      LF = 10
+      SPACE = 32
+
+      attr_reader :last_event_id, :reconnection_time
+
+      def initialize
+        @buffer = String.new(encoding: Encoding::BINARY)
+        @position = 0
+        @searched = 0
+        @data = +''
+        @event_type = +''
+        @id_buffer = +''
+        @last_event_id = +''
+        @reconnection_time = nil
+        @started = false
+        @after_cr = false
+      end
+
+      def feed(chunk, &)
+        compact
+        @buffer << (chunk.encoding == Encoding::BINARY ? chunk : chunk.b)
+        return unless start
+
+        skip_line_feed_after_carriage_return
+        while (index = @buffer.index(LINE_BREAK, @searched))
+          line = @buffer.byteslice(@position, index - @position)
+          @position = @searched = line_end(index)
+          process_line(line, &)
+        end
+        @searched = @buffer.bytesize
+      end
+
+      private
+
+      def compact
+        return if @position.zero?
+
+        @buffer = @buffer.byteslice(@position, @buffer.bytesize - @position)
+        @searched -= @position
+        @position = 0
+      end
+
+      # The stream decodes as UTF-8, which drops one byte order mark at its
+      # very start, so the first bytes wait until they can be told apart.
+      def start
+        return true if @started
+        return false if @buffer.bytesize < BOM.bytesize && BOM.start_with?(@buffer)
+
+        @position = @searched = BOM.bytesize if @buffer.start_with?(BOM)
+        @started = true
+      end
+
+      # A carriage return that ended the previous piece may be the first
+      # half of a CRLF pair.
+      def skip_line_feed_after_carriage_return
+        return unless @after_cr && @buffer.bytesize > @position
+
+        @after_cr = false
+        @position = @searched = @position + 1 if @buffer.getbyte(@position) == LF
+      end
+
+      def line_end(index)
+        return index + 1 unless @buffer.getbyte(index) == CR
+        return index + 2 if @buffer.getbyte(index + 1) == LF
+
+        @after_cr = index + 1 == @buffer.bytesize
+        index + 1
+      end
+
+      def process_line(line, &)
+        return dispatch(&) if line.empty?
+        return if line.start_with?(':')
+
+        colon = line.index(':')
+        return process_field(line, +'') unless colon
+
+        offset = line.getbyte(colon + 1) == SPACE ? colon + 2 : colon + 1
+        process_field(line.byteslice(0, colon), line.byteslice(offset, line.bytesize - offset))
+      end
+
+      def process_field(field, value)
+        case field
+        when 'event' then @event_type = decode(value)
+        when 'data' then @data << decode(value) << "\n"
+        when 'id' then @id_buffer = decode(value) unless value.include?("\0")
+        when 'retry' then @reconnection_time = value.to_i if value.match?(DIGITS)
+        end
+      end
+
+      def dispatch
+        @last_event_id = @id_buffer
+        data = @data
+        type = @event_type
+        @data = +''
+        @event_type = +''
+        return if data.empty?
+
+        data.delete_suffix!("\n")
+        yield type.empty? ? 'message' : type, data, @last_event_id
+      end
+
+      def decode(bytes)
+        text = bytes.force_encoding(Encoding::UTF_8)
+        text.valid_encoding? ? text : text.scrub
+      end
+    end
+  end
+end

data/lib/ruby_llm/transport/json_response.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'json'
+
+module RubyLLM
+  module Transport
+    class JsonResponse < Faraday::Middleware # :nodoc:
+      def initialize(app, parser_options: {}, preserve_raw: false)
+        super(app)
+        @parser_options = parser_options || {}
+        @preserve_raw = preserve_raw
+      end
+
+      def on_complete(env)
+        body = env[:body]
+        return unless body.respond_to?(:to_str) && json_response?(env)
+
+        env[:raw_body] = body if @preserve_raw
+        env[:body] = body.strip.empty? ? nil : JSON.parse(body, **@parser_options)
+      rescue StandardError, SyntaxError => e
+        raise Faraday::ParsingError.new(e, env[:response])
+      end
+
+      private
+
+      def json_response?(env)
+        content_type = env[:response_headers]['content-type'].to_s.split(';', 2).first
+        content_type&.match?(/\bjson$/)
+      end
+    end
+  end
+end

data/lib/ruby_llm/transport/signing_middleware.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'faraday'
+
+module RubyLLM
+  module Transport # :nodoc:
+    # Sits inside Faraday retry middleware so every transport attempt is
+    # signed when it is sent, over the body it sends. A signature computed
+    # once before the first attempt can expire before a retry goes out.
+    class SigningMiddleware < Faraday::Middleware # :nodoc: all
+      CONTEXT_KEY = :ruby_llm_signer
+
+      def call(env)
+        signer = env.request.context&.[](CONTEXT_KEY)
+        sign(env.request_headers, signer.call(env)) if signer
+        @app.call(env)
+      end
+
+      private
+
+      # A nil value removes a header an earlier attempt was signed with.
+      def sign(headers, signed)
+        signed.each { |name, value| value.nil? ? headers.delete(name) : headers[name] = value }
+      end
+    end
+  end
+end
+
+Faraday::Middleware.register_middleware(llm_signing: RubyLLM::Transport::SigningMiddleware)

data/lib/ruby_llm/version.rb

@@ -2,5 +2,5 @@
 
 module RubyLLM
   # The version of the ruby_llm gem, as a string.
-  VERSION = '2.0.0'
+  VERSION = '2.1.0'
 end

data/lib/ruby_llm/video.rb

@@ -49,7 +49,8 @@ module RubyLLM
     # vocabulary, such as durations and resolutions, and merges them into
     # the request as-is. +context:+ supplies a Context whose configuration
     # replaces the global one. +metadata:+ is included in the
-    # instrumentation payload.
+    # instrumentation payload. +owner:+ attributes the usage to a record,
+    # such as a user, and wins over RubyLLM.with_usage_owner.
     #
     #   video = RubyLLM.animate("a hummingbird in slow motion", model: "veo-3.1-fast-generate-preview")
     #
@@ -68,13 +69,14 @@ module RubyLLM
                      with: nil,
                      extend: nil,
                      provider_options: {},
-                     metadata: nil)
+                     metadata: nil,
+                     owner: nil)
       config = context&.config || RubyLLM.config
       payload = { model:, prompt:, provider_options:, metadata: }
 
       RubyLLM.instrument('video.ruby_llm', payload, config: config) do |event|
         job = VideoJob.animate_later(prompt, model:, provider:, assume_model_exists:,
-                                             context:, with:, extend:, provider_options:, metadata:)
+                                             context:, with:, extend:, provider_options:, metadata:, owner:)
         event[:model] = job.model
         event[:job_id] = job.id
         job.wait

data/lib/ruby_llm/video_job.rb

@@ -27,9 +27,20 @@ module RubyLLM
     # The provider's raw response from the last submit or poll.
     attr_reader :raw
 
+    # The clip length in seconds the request asked for, or +nil+ when it
+    # left the length to the provider's default.
+    attr_reader :duration
+
+    # The resolution the request asked for, as the provider spells it,
+    # such as <tt>"720p"</tt> or <tt>"1280x720"</tt>, or +nil+ when it left
+    # the resolution to the provider's default.
+    attr_reader :resolution
+
     # Submits a video generation job and returns a VideoJob without
     # waiting for the result. Most code calls this through
     # RubyLLM.animate_later. Takes the same arguments as Video.animate.
+    # The job records its usage, attributed to +owner:+ or the owner of
+    # RubyLLM.with_usage_owner, once it finishes.
     #
     #   job = RubyLLM.animate_later("a hummingbird in slow motion")
     #   job.id      # => "0eb6910f-a353-4699-9d1e-6a4f7a5b39e2"
@@ -43,7 +54,8 @@ module RubyLLM
                            with: nil,
                            extend: nil,
                            provider_options: {},
-                           metadata: nil)
+                           metadata: nil,
+                           owner: nil)
       config = context&.config || RubyLLM.config
       model ||= config.default_video_model
       model, provider_instance = Models.resolve(model, provider: provider, assume_model_exists: assume_model_exists,
@@ -59,19 +71,27 @@ module RubyLLM
       }
 
       RubyLLM.instrument('video_job.ruby_llm', payload, config: config) do |event|
-        job = provider_instance.animate_later(prompt, model:, with:, extend:, provider_options:)
+        job = Accounting::Usage.owned_by(owner) do
+          provider_instance.animate_later(prompt, model:, with:, extend:, provider_options:)
+        end
         event[:job_id] = job.id
         job
       end
     end
 
-    def initialize(id:, protocol:, model: nil, status: :pending, raw: nil, error: nil) # :nodoc:
+    def initialize(id:, protocol:, model: nil, status: :pending, raw: nil, error: nil, # :nodoc:
+                   reported_cost: nil, duration: nil, resolution: nil)
       @id = id
       @protocol = protocol
       @model = model
       @status = status
       @raw = raw
       @error = error
+      @reported_cost = reported_cost
+      @duration = duration
+      @resolution = resolution
+      @usage_owner = Accounting::Usage.owner
+      record_usage
     end
 
     # Returns +true+ while the provider is still rendering the video.
@@ -104,9 +124,22 @@ module RubyLLM
       @status = state.fetch(:status)
       @raw = state[:raw]
       @error = state[:error]
+      @reported_cost = state[:reported_cost]
+      record_usage
       self
     end
 
+    # Returns the Cost the provider reported for the job. Its total is
+    # +nil+ while the job is pending or when the provider reports no
+    # price.
+    #
+    #   job.wait
+    #   job.cost.total # => 0.05
+    #
+    def cost
+      Cost.new(tokens: Tokens.new(reported_cost: @reported_cost))
+    end
+
     # Polls the job until it finishes, then returns self. Raises Error
     # when the job fails or +timeout+ elapses first. +timeout+ and
     # +interval+ are seconds and default to the configured
@@ -117,9 +150,10 @@ module RubyLLM
       deadline = monotonic_time + timeout
 
       until done?
-        raise Error, "Video generation timed out after #{timeout} seconds" if monotonic_time > deadline
+        remaining = deadline - monotonic_time
+        raise Error, "Video generation timed out after #{timeout} seconds" unless remaining.positive?
 
-        sleep interval
+        sleep [interval, remaining].min
         refresh
       end
       raise Error, "Video generation failed: #{error}" if failed?
@@ -139,6 +173,15 @@ module RubyLLM
 
     private
 
+    def record_usage
+      return if pending? || @usage_recorded
+
+      @usage_recorded = true
+      entry = Accounting::Usage::Entry.new(operation: :video, provider: @protocol.provider.slug, model:,
+                                           status: completed? ? :succeeded : :failed, cost:, owner: @usage_owner)
+      Accounting::Usage.report(entry, config: @protocol.config)
+    end
+
     def monotonic_time
       Process.clock_gettime(Process::CLOCK_MONOTONIC)
     end