ruby_llm 2.0.0 → 2.1.0

data/lib/ruby_llm/choice.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  # A selected judgment option and its full probability distribution.
+  class Choice
+    include Support::Inspectable
+
+    # Returns the selected option, preserving its declared String or Symbol type.
+    attr_reader :choice
+
+    # Returns each declared option's probability.
+    attr_reader :probabilities
+
+    # Returns the provider's confidence in the distribution, between 0 and 1.
+    attr_reader :confidence
+
+    def initialize(choice:, probabilities:, confidence:) # :nodoc:
+      @choice = choice.is_a?(String) ? choice.dup.freeze : choice
+      @probabilities = probabilities.dup.freeze
+      @confidence = confidence
+      freeze
+    end
+
+    # Returns :choice.
+    def type
+      :choice
+    end
+
+    # Returns the answer as a Hash.
+    def to_h
+      { type:, choice:, probabilities:, confidence: }
+    end
+
+    def inspect_attributes # :nodoc:
+      { choice:, confidence: }
+    end
+  end
+end

data/lib/ruby_llm/configuration.rb

@@ -82,6 +82,13 @@ module RubyLLM
     # Default: <tt>'omni-moderation-latest'</tt>.
     option :default_moderation_model, 'omni-moderation-latest'
 
+    ##
+    # :attr_accessor: default_judgment_model
+    #
+    # The model id used by RubyLLM.judge and Judge when no model is given.
+    # Default: <tt>'jev-latest'</tt>.
+    option :default_judgment_model, 'jev-latest'
+
     ##
     # :attr_accessor: default_image_model
     #
@@ -152,6 +159,33 @@ module RubyLLM
     # Optional persistence adapter installed by the Rails integration.
     option :batch_store, nil # :nodoc:
 
+    ##
+    # :attr_accessor: mcp_credential_store
+    #
+    # Store for the OAuth credentials of MCP servers. Rails apps keep them
+    # in the +ruby_llm_mcp_credentials+ table automatically. A store
+    # responds to +read(key)+, +write(key, data, owner:)+, and
+    # +delete(key)+. A store that several processes share also responds
+    # to +synchronize(key)+, running the block while no other process
+    # holds +key+, so only one of them refreshes a token at a time.
+    # Default: +nil+ (keep them in memory).
+    option :mcp_credential_store, nil
+
+    ##
+    # :attr_accessor: mcp_client_name
+    #
+    # The client name RubyLLM gives when it registers with an MCP server's
+    # authorization server. Default: <tt>"RubyLLM"</tt>.
+    option :mcp_client_name, 'RubyLLM'
+
+    ##
+    # :attr_accessor: mcp_client_id
+    #
+    # The HTTPS URL of your OAuth client metadata document. Authorization
+    # servers that support client ID metadata documents use it instead of
+    # registering RubyLLM. Default: +nil+.
+    option :mcp_client_id, nil
+
     # Keeps custom 1.x initializers bootable long enough to run the 2.0
     # upgrade generator. The registry is always RubyLLM-owned in 2.0.
     def model_registry_class=(_value) # :nodoc:
@@ -288,9 +322,8 @@ module RubyLLM
     #
     # Timeout in seconds for the regular expressions that scrub logged
     # payloads. Defaults to the global <tt>Regexp.timeout</tt>, or 1.0
-    # when none is set. Requires Ruby 3.2 or later; on older Rubies
-    # setting a value logs a warning.
-    option :log_regexp_timeout, -> { Regexp.respond_to?(:timeout) ? (Regexp.timeout || 1.0) : nil }
+    # when none is set.
+    option :log_regexp_timeout, -> { Regexp.timeout || 1.0 }
 
     def initialize # :nodoc:
       self.class.send(:defaults).each do |key, default|
@@ -311,14 +344,6 @@ module RubyLLM
       "#<#{self.class.name} #{attributes.join(', ')}>"
     end
 
-    remove_method :log_regexp_timeout=
-    def log_regexp_timeout=(value) # :nodoc:
-      if value && !Regexp.respond_to?(:timeout)
-        RubyLLM.logger.warn("log_regexp_timeout is not supported on Ruby #{RUBY_VERSION}")
-      end
-      @log_regexp_timeout = value
-    end
-
     private
 
     # Whether the +name+ environment variable is set to something that means

data/lib/ruby_llm/context.rb

@@ -35,8 +35,8 @@ module RubyLLM
 
     # Creates a new Chat that uses this context's configuration.
     # Accepts the same arguments as RubyLLM.chat.
-    def chat(*args, **kwargs, &)
-      Chat.new(*args, **kwargs, context: self, &)
+    def chat(*, **, &)
+      Chat.new(*, **, context: self, &)
     end
 
     # Counts tokens using this context's configuration.
@@ -47,8 +47,8 @@ module RubyLLM
 
     # Tokenizes plain text using this context's configuration.
     # Accepts the same arguments as RubyLLM.tokenize.
-    def tokenize(*args, **kwargs)
-      Tokenization.tokenize(*args, **kwargs, context: self)
+    def tokenize(*, **)
+      Tokenization.tokenize(*, **, context: self)
     end
 
     # Runs a named workflow using this context's instrumenter. Accepts the same
@@ -59,8 +59,8 @@ module RubyLLM
 
     # Generates embeddings using this context's configuration.
     # Accepts the same arguments as RubyLLM.embed.
-    def embed(*args, **kwargs, &)
-      Embedding.embed(*args, **kwargs, context: self, &)
+    def embed(*, **, &)
+      Embedding.embed(*, **, context: self, &)
     end
 
     # Stages an embedding request using this context's configuration.
@@ -71,81 +71,94 @@ module RubyLLM
 
     # Generates an image using this context's configuration.
     # Accepts the same arguments as RubyLLM.paint.
-    def paint(*args, **kwargs, &)
-      Image.paint(*args, **kwargs, context: self, &)
+    def paint(*, **, &)
+      Image.paint(*, **, context: self, &)
     end
 
     # Generates a video using this context's configuration, blocking
     # until it is ready. Accepts the same arguments as RubyLLM.animate.
-    def animate(*args, **kwargs, &)
-      Video.animate(*args, **kwargs, context: self, &)
+    def animate(*, **, &)
+      Video.animate(*, **, context: self, &)
     end
 
     # Submits a video generation job using this context's configuration.
     # Accepts the same arguments as RubyLLM.animate_later.
-    def animate_later(*args, **kwargs, &)
-      VideoJob.animate_later(*args, **kwargs, context: self, &)
+    def animate_later(*, **, &)
+      VideoJob.animate_later(*, **, context: self, &)
     end
 
     # Runs content moderation using this context's configuration.
     # Accepts the same arguments as RubyLLM.moderate.
-    def moderate(*args, **kwargs, &)
-      Moderation.moderate(*args, **kwargs, context: self, &)
+    def moderate(*, **, &)
+      Moderation.moderate(*, **, context: self, &)
+    end
+
+    # Judges text, structured data, or images using this context's configuration.
+    # Accepts the same arguments as RubyLLM.judge.
+    def judge(*, **, &)
+      Judge.judge(*, **, context: self, &)
     end
 
     # Runs hosted research using this context's configuration.
     # Accepts the same arguments as RubyLLM.research.
-    def research(*args, **kwargs)
-      ResearchJob.research(*args, **kwargs, context: self)
+    def research(*, **)
+      ResearchJob.research(*, **, context: self)
     end
 
     # Submits hosted research using this context's configuration.
     # Accepts the same arguments as RubyLLM.research_later.
-    def research_later(*args, **kwargs)
-      ResearchJob.research_later(*args, **kwargs, context: self)
+    def research_later(*, **)
+      ResearchJob.research_later(*, **, context: self)
     end
 
     # Generates speech audio using this context's configuration. Given a
     # block, yields SpeechChunk objects and returns the complete Speech.
     # Accepts the same arguments as RubyLLM.speak.
-    def speak(*args, **kwargs, &)
-      Speech.speak(*args, **kwargs, context: self, &)
+    def speak(*, **, &)
+      Speech.speak(*, **, context: self, &)
     end
 
     # Transcribes audio using this context's configuration.
     # Accepts the same arguments as RubyLLM.transcribe.
-    def transcribe(*args, **kwargs, &)
-      Transcription.transcribe(*args, **kwargs, context: self, &)
+    def transcribe(*, **, &)
+      Transcription.transcribe(*, **, context: self, &)
     end
 
     # Extracts document text using this context's configuration.
     # Accepts the same arguments as RubyLLM.ocr.
-    def ocr(*args, **kwargs, &)
-      OCR.ocr(*args, **kwargs, context: self, &)
+    def ocr(*, **, &)
+      OCR.ocr(*, **, context: self, &)
     end
 
     # Ranks documents using this context's configuration.
     # Accepts the same arguments as RubyLLM.rerank.
-    def rerank(*args, **kwargs, &)
-      Rerank.rerank(*args, **kwargs, context: self, &)
+    def rerank(*, **, &)
+      Rerank.rerank(*, **, context: self, &)
     end
 
     # Uploads a file to a provider using this context's configuration.
     # Accepts the same arguments as RubyLLM.upload.
-    def upload(*args, **kwargs, &)
-      UploadedFile.upload(*args, **kwargs, context: self, &)
+    def upload(*, **, &)
+      UploadedFile.upload(*, **, context: self, &)
     end
 
     # Downloads a provider-hosted file using this context's configuration.
     # Accepts the same arguments as RubyLLM.download.
-    def download(*args, **kwargs, &)
-      UploadedFile.download(*args, **kwargs, context: self, &)
+    def download(*, **, &)
+      UploadedFile.download(*, **, context: self, &)
     end
 
     # Creates a provider-side prompt cache using this context's
     # configuration. Accepts the same arguments as RubyLLM.cache.
-    def cache(*args, **kwargs, &)
-      CachedContent.create(*args, **kwargs, context: self, &)
+    def cache(*, **, &)
+      CachedContent.create(*, **, context: self, &)
+    end
+
+    # Connects to an MCP server using this context's configuration, so its
+    # connection settings apply to the server's requests and to its OAuth
+    # requests. Accepts the same arguments as RubyLLM.mcp.
+    def mcp(...)
+      MCP.define(...).new(context: self)
     end
   end
 end

data/lib/ruby_llm/cost.rb

@@ -130,7 +130,9 @@ module RubyLLM
     end
 
     # Returns the cost of cache-write input tokens in US dollars, or +nil+
-    # when the token count or its pricing is unavailable.
+    # when the token count or its pricing is unavailable. Writes with a
+    # longer lifetime, reported in Tokens#cache_write_by_ttl, are priced at
+    # the provider's rate for that lifetime.
     def cache_write
       @amounts[:cache_write]
     end
@@ -199,6 +201,7 @@ module RubyLLM
 
     def missing_component?(component)
       return image_input_missing? if component == :input && detailed_image_input?
+      return cache_write_missing? if component == :cache_write && repriced_cache_writes?
       return false if component == :thinking && !thinking_priced_separately?
 
       tokens_for(component).to_i.positive? && price_for(component).nil?
@@ -206,6 +209,7 @@ module RubyLLM
 
     def amount_for(component)
       return image_input_amount if component == :input && detailed_image_input?
+      return cache_write_amount if component == :cache_write && repriced_cache_writes?
 
       token_count = tokens_for(component)
       return nil if token_count.nil?
@@ -299,6 +303,43 @@ module RubyLLM
       pricing.public_send(@category)
     end
 
+    def repriced_cache_writes?
+      cache_write_multipliers.any?
+    end
+
+    def cache_write_amount
+      cache_write_parts.sum { |count, price| count * price / PER_MILLION } unless cache_write_missing?
+    end
+
+    def cache_write_missing?
+      cache_write_parts.any? { |_, price| price.nil? }
+    end
+
+    # Cache writes as [count, price] pairs: lifetimes the provider prices
+    # from the input price, then the rest at the registry's cache-write price.
+    def cache_write_parts
+      parts = cache_write_multipliers.map do |ttl, multiplier|
+        [cache_write_by_ttl[ttl], input_price && (input_price * multiplier)]
+      end
+      standard = @tokens.cache_write.to_i - parts.sum(&:first)
+      parts << [standard, price_for(:cache_write)] if standard.positive?
+      parts
+    end
+
+    def cache_write_multipliers
+      @cache_write_multipliers ||= begin
+        provider = @model.provider_class if @model.respond_to?(:provider_class)
+        cache_write_by_ttl.filter_map do |ttl, _|
+          multiplier = provider&.cache_write_input_multiplier(@model.id, ttl)
+          [ttl, multiplier] if multiplier
+        end.to_h
+      end
+    end
+
+    def cache_write_by_ttl
+      (@tokens.cache_write_by_ttl if @tokens.respond_to?(:cache_write_by_ttl)) || {}
+    end
+
     def image_cost?
       %i[image images].include?(@category)
     end

data/lib/ruby_llm/embedding.rb

@@ -54,6 +54,8 @@ module RubyLLM
       Cost.new(tokens:, model: model_info, category: :embeddings)
     end
 
+    attr_writer :model_info # :nodoc:
+
     def model_info # :nodoc:
       @model_info ||= RubyLLM.models.find(model)
     rescue ModelNotFoundError
@@ -97,6 +99,8 @@ module RubyLLM
     # in the provider's request vocabulary and merges them into the
     # request as-is. +metadata:+ is not sent to the provider; it is
     # attached to the emitted +embedding.ruby_llm+ instrumentation event.
+    # +owner:+ attributes the usage to a record, such as a user, and wins
+    # over RubyLLM.with_usage_owner.
     def self.embed(text,
                    model: nil,
                    provider: nil,
@@ -107,7 +111,8 @@ module RubyLLM
                    title: nil,
                    with: nil,
                    provider_options: {},
-                   metadata: nil)
+                   metadata: nil,
+                   owner: nil)
       config = context&.config || RubyLLM.config
       model ||= config.default_embedding_model
       model, provider_instance = Models.resolve(model, provider: provider, assume_model_exists: assume_model_exists,
@@ -132,7 +137,9 @@ module RubyLLM
       }
 
       RubyLLM.instrument('embedding.ruby_llm', payload, config: config) do |event|
-        result = provider_instance.embed(text, model:, dimensions:, task_type:, title:, with:, provider_options:)
+        result = Accounting::Usage.owned_by(owner) do
+          provider_instance.embed(text, model:, dimensions:, task_type:, title:, with:, provider_options:)
+        end
         event[:result] = result
         event[:response_model] = result.model
         event[:tokens] = result.tokens

data/lib/ruby_llm/error.rb

@@ -4,8 +4,9 @@ module RubyLLM
   # Error is the base class for provider-operation errors raised by
   # RubyLLM, including API, network, capability, and provider response-shape
   # failures. When an HTTP response is available, it wraps that response and
-  # normalizes the message across providers. Subclasses map common HTTP
-  # status codes: BadRequestError (400), UnauthorizedError (401),
+  # normalizes the message across providers. When the provider refuses a
+  # conversation, #request_shape describes what it was sent. Subclasses map
+  # common HTTP status codes: BadRequestError (400), UnauthorizedError (401),
   # PaymentRequiredError (402), ForbiddenError (403), RateLimitError (429),
   # ServerError (500), ServiceUnavailableError (502 to 504), and
   # OverloadedError (529).
@@ -27,6 +28,13 @@ module RubyLLM
     # available. Its +status+ and +body+ carry the provider's reply.
     attr_reader :response
 
+    # The Protocol that rendered the request the error answered, which
+    # reads its payload back for #request_shape.
+    attr_accessor :request_protocol # :nodoc:
+
+    # The payload request_protocol rendered for the request.
+    attr_accessor :request_payload # :nodoc:
+
     def self.default_message # :nodoc:
       nil
     end
@@ -37,6 +45,24 @@ module RubyLLM
       @response = response
       super(message || response&.body || self.class.default_message)
     end
+
+    # Returns a RequestShape describing the conversation request the error
+    # answered: each turn's parts and their sizes, never their contents,
+    # and the problems providers are known to refuse. Returns +nil+ for
+    # errors raised before a request goes out, such as an unsupported
+    # attachment, and for operations that send no conversation, such as
+    # embeddings.
+    #
+    #   rescue RubyLLM::BadRequestError => e
+    #     Rails.logger.error("#{e.message}\n#{e.request_shape}")
+    #     Rails.error.report(e, context: { request_shape: e.request_shape&.to_h })
+    #
+    def request_shape
+      return unless request_protocol
+      return @request_shape if defined?(@request_shape)
+
+      @request_shape = request_protocol.request_shape(request_payload)
+    end
   end
 
   # Raised when a deprecated API is used and
@@ -137,6 +163,12 @@ module RubyLLM
     end
   end
 
+  # Raised when the provider's content filters block a transcription and
+  # no transcript comes back. The message names the provider's reason,
+  # such as +SAFETY+. Chat responses report a block through
+  # Message#content_filtered? instead of raising.
+  class ContentFilterError < Error; end
+
   # Raised for HTTP 529 responses when the provider is temporarily
   # overloaded.
   class OverloadedError < Error

data/lib/ruby_llm/evaluation/assertions.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class Evaluation
+    class Assertions # :nodoc:
+      METHOD = /\A(?:assert|refute)(?:_|\z)/
+
+      # Matches Minitest assertion failures without loading Minitest.
+      module Failure # :nodoc:
+        def self.===(error)
+          defined?(::Minitest::Assertion) && error.is_a?(::Minitest::Assertion)
+        end
+      end
+
+      attr_accessor :assertions
+
+      class << self
+        def method?(name)
+          METHOD.match?(name) && available? && ::Minitest::Assertions.method_defined?(name)
+        end
+
+        def available?
+          require 'minitest'
+          true
+        rescue LoadError
+          false
+        end
+      end
+
+      def initialize
+        raise LoadError, "Evaluation assertions require the 'minitest' gem. Add it to your Gemfile." unless
+          self.class.available?
+
+        extend ::Minitest::Assertions
+
+        @assertions = 0
+      end
+    end
+  end
+end

data/lib/ruby_llm/evaluation/case.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class Evaluation
+    # One dataset example, with application inputs and optional reference evidence.
+    class Case
+      include Support::Inspectable
+
+      # Returns the stable case name used to compare runs.
+      attr_reader :name
+      # Returns the value passed to Evaluation#perform.
+      attr_reader :inputs
+      # Returns the reference answer, or nil when none was supplied.
+      attr_reader :expected_output
+      # Returns application-defined reference evidence and category labels.
+      attr_reader :metadata
+
+      # Creates a case. Inputs and reference evidence must be JSON-compatible.
+      def initialize(name:, inputs:, metadata: {}, **reference)
+        unless (reference.keys - [:expected_output]).empty?
+          raise ArgumentError,
+                "Unknown case fields: #{reference.keys - [:expected_output]}"
+        end
+        raise ArgumentError, 'A case name cannot be empty' if name.to_s.empty?
+        raise ArgumentError, 'Case metadata must be a Hash' unless metadata.is_a?(Hash)
+
+        @name = name.to_s.dup.freeze
+        @inputs = Judge::Data.copy(inputs)
+        @metadata = Judge::Data.copy(metadata)
+        @expected_output = Judge::Data.copy(reference[:expected_output])
+        @reference_supplied = reference.key?(:expected_output)
+        freeze
+      end
+
+      # Returns whether a reference was supplied, including an explicit nil.
+      def expected_output?
+        @reference_supplied
+      end
+
+      # Returns the portable dataset representation of this case.
+      def to_h
+        data = { name:, inputs:, metadata: }
+        data[:expected_output] = expected_output if expected_output?
+        data
+      end
+
+      def inspect_attributes # :nodoc:
+        { name: }
+      end
+    end
+  end
+end

data/lib/ruby_llm/evaluation/dataset.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'pathname'
+
+module RubyLLM
+  class Evaluation
+    module Dataset # :nodoc:
+      module_function
+
+      def load(source, name:)
+        source = rows(source, name:)
+        cases = source.map do |row|
+          row.is_a?(Case) ? row : Case.new(**row.transform_keys(&:to_sym))
+        end
+        raise ArgumentError, 'A dataset cannot be empty' if cases.empty?
+        raise ArgumentError, 'Dataset case names must be unique' unless cases.map(&:name).uniq.size == cases.size
+
+        cases
+      end
+
+      def rows(source, name:)
+        source = source.call if source.respond_to?(:call)
+        source ||= discover(name)
+        source = read(source) if source.is_a?(String) || source.is_a?(Pathname)
+        source = cases_from_hash(source) if source.is_a?(Hash)
+        raise ArgumentError, 'A dataset must contain enumerable cases' unless source.respond_to?(:map)
+
+        source
+      end
+
+      def cases_from_hash(source)
+        unknown = source.keys.map(&:to_s) - %w[name cases]
+        if unknown.any?
+          raise ArgumentError,
+                "Unsupported dataset fields: #{unknown.join(', ')}; declare evaluators in Ruby"
+        end
+
+        source.fetch('cases') { source.fetch(:cases) }
+      end
+
+      def discover(name)
+        raise ArgumentError, 'An anonymous evaluation needs an explicit dataset' unless name
+
+        filename = name.gsub('::', '/').gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+                       .gsub(/([a-z\d])([A-Z])/, '\1_\2').downcase
+        root = Prompt.root.parent.join('evals')
+        paths = %w[yml yaml json jsonl].map { |extension| root.join("#{filename}.#{extension}") }.select(&:file?)
+        raise ArgumentError, "Dataset not found: #{root.join(filename)}.{yml,yaml,json,jsonl}" if paths.empty?
+        raise ArgumentError, "Ambiguous dataset: #{paths.join(', ')}" if paths.size > 1
+
+        paths.first
+      end
+
+      def read(path)
+        case File.extname(path)
+        when '.yml', '.yaml' then YAML.safe_load_file(path, aliases: false)
+        when '.json' then JSON.parse(File.read(path))
+        when '.jsonl' then File.foreach(path).reject { |line| line.strip.empty? }.map { |line| JSON.parse(line) }
+        else raise ArgumentError, "Unsupported dataset format: #{path}"
+        end
+      end
+    end
+  end
+end