ruby_llm 2.0.0 → 2.1.0

data/lib/ruby_llm/evaluation/trial.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class Evaluation
+    # The recorded outcome of one case and repetition, including failed attempts.
+    class Trial
+      include Support::Inspectable
+      include Accounting::Usage::Result
+
+      # Returns the case whose inputs were executed.
+      attr_reader :test_case
+      # Returns the one-based repetition number.
+      attr_reader :repetition
+      # Returns the original value returned by perform, when it completed.
+      attr_reader :result
+      # Returns the primary output extracted from the returned value.
+      attr_reader :output
+      # Returns the frozen evidence supplied to evaluators.
+      attr_reader :evidence
+      # Returns each criterion's Evaluation::Result.
+      attr_reader :evaluations
+      # Returns the number of Ruby assertions attempted.
+      attr_reader :assertion_count
+      # Returns an assertion failure, or nil.
+      attr_reader :assertion_failure
+      # Returns an execution or evidence-conversion error, or nil.
+      attr_reader :error
+      # Returns elapsed wall-clock seconds, including evaluators.
+      attr_reader :duration
+
+      def initialize(test_case:, repetition:, task_usage: [], evaluator_usage: [], **attributes) # :nodoc:
+        @test_case = test_case
+        @repetition = repetition
+        @task_usage = task_usage.freeze
+        @evaluator_usage = evaluator_usage.freeze
+        @ruby_llm_usage_entries = (task_usage + evaluator_usage).freeze
+        attributes.each { |key, value| instance_variable_set("@#{key}", value) }
+        @evaluations = Array(@evaluations).freeze
+        freeze
+      end
+
+      # Returns a Tokens aggregate for every provider attempt made during this trial.
+      def tokens
+        ruby_llm_usage_tokens
+      end
+
+      # Returns a Cost aggregate for this trial, including task and evaluator requests.
+      def cost
+        ruby_llm_usage_cost
+      end
+
+      # Returns the task's tokens, including setup, assertions, and teardown requests.
+      def task_tokens
+        Tokens.aggregate(@task_usage.map(&:tokens))
+      end
+
+      # Returns the tokens used by the evaluators and their tools.
+      def evaluator_tokens
+        Tokens.aggregate(@evaluator_usage.map(&:tokens))
+      end
+
+      # Returns the task's cost, including setup, assertions, and teardown requests.
+      def task_cost
+        Cost.aggregate(@task_usage.map(&:cost), complete: @task_usage.all?(&:cost_available?))
+      end
+
+      # Returns the cost of the evaluators and their tools.
+      def evaluator_cost
+        Cost.aggregate(@evaluator_usage.map(&:cost), complete: @evaluator_usage.all?(&:cost_available?))
+      end
+
+      # Returns :passed, :failed, :measured, :unassessed, or :error. Empty trials are never passes.
+      def status
+        statuses = evaluations.map(&:status)
+        statuses << :error if error
+        statuses << :failed if assertion_failure
+        statuses << :measured if evaluations.empty? && assertion_count.to_i.zero?
+        %i[error failed unassessed measured].find { |status| statuses.include?(status) } || :passed
+      end
+
+      # Returns whether every assessment passed and no error occurred.
+      def passed?
+        status == :passed
+      end
+
+      # Returns the portable trial record, excluding the live application object.
+      def to_h
+        { case: test_case.to_h, repetition:, status:, evidence:, evaluations: evaluations.map(&:to_h),
+          assertion_count:, assertion_failure: assertion_failure&.message, duration:,
+          error: error && { class: error.class.name, message: error.message },
+          tokens: tokens.to_h, cost: cost.to_h,
+          task_tokens: task_tokens.to_h, evaluator_tokens: evaluator_tokens.to_h,
+          task_cost: task_cost.to_h, evaluator_cost: evaluator_cost.to_h }
+      end
+
+      def inspect_attributes # :nodoc:
+        { case: test_case.name, repetition:, status: }
+      end
+    end
+  end
+end

data/lib/ruby_llm/evaluation.rb

@@ -0,0 +1,289 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  # Defines reusable evaluations with a dataset, semantic criteria, and Ruby assertions.
+  # Datasets default to app/evals/<class_name>.yml, .yaml, .json, or .jsonl.
+  # Without declared criteria, evaluates correctness against each case's expected_output.
+  #
+  #   class SupportEvaluation < RubyLLM::Evaluation
+  #     def perform(input)
+  #       SupportAgent.new.ask(input)
+  #     end
+  #   end
+  #
+  #   SupportEvaluation.run.save("tmp/support.json")
+  class Evaluation
+    CORRECTNESS = 'The answer agrees with the expected output'
+    private_constant :CORRECTNESS
+
+    class << self
+      def inherited(subclass) # :nodoc:
+        super
+        subclass.instance_variable_set(:@dataset, @dataset)
+        subclass.instance_variable_set(:@evaluator, @evaluator)
+        subclass.instance_variable_set(:@definitions, definitions.dup)
+        subclass.instance_variable_set(:@adapters, adapters.dup)
+      end
+
+      # Sets the dataset path, enumerable of cases, or block returning cases.
+      # With no argument, returns the configured source. The default is discovered by class name.
+      def dataset(source = nil, &block)
+        return @dataset if source.nil? && !block
+
+        raise ArgumentError, 'Pass a dataset or a block, not both' if source && block
+
+        @dataset = block || source
+      end
+
+      # Sets the default evaluator: a model, Agent class, or Judge class.
+      # Model keywords accept provider, protocol, and context as Chat does.
+      # Without a declaration, uses the default chat model and built-in reviewer.
+      # Pass false to disable model grading and run only Ruby assertions.
+      def evaluator(target = nil, **options)
+        if target.nil? && options.empty?
+          @evaluator = Evaluator.new if @evaluator.nil?
+          return @evaluator
+        end
+        raise ArgumentError, 'A disabled evaluator cannot have model options' if target == false && options.any?
+
+        @evaluator = target == false ? false : Evaluator.new(target, **options)
+      end
+
+      # Declares a semantic criterion. A minimum applies to a native probability or score;
+      # without one, numeric decisions are measured but do not count as passes.
+      # Omit instructions to set a minimum on a question already defined by a Judge.
+      # Declared criteria replace implicit correctness. Use evaluation :correctness
+      # without instructions to include the built-in reference comparison explicitly.
+      # Supply evaluator to override the class evaluator for this criterion.
+      def evaluation(name, instructions = nil, minimum: nil, evaluator: nil)
+        validate_criterion(name, minimum)
+        name = name.to_sym
+        @declared_names ||= []
+        raise ArgumentError, "Duplicate evaluation: #{name}" if @declared_names.include?(name)
+
+        @declared_names << name
+        backend = Evaluator.new(evaluator) if evaluator
+        definitions[name] = { name:, instructions: instructions&.dup&.freeze, minimum:, evaluator: backend }.freeze
+      end
+
+      # Converts a custom result into evaluation evidence. The original stays available as result.
+      #
+      #   adapt Invoice do |invoice|
+      #     invoice.attributes.slice("total", "currency")
+      #   end
+      def adapt(type, &block)
+        raise ArgumentError, 'An adapter needs a class and a block' unless type.is_a?(Module) && block
+
+        adapters[type] = block
+      end
+
+      # Returns the dataset cases without running the application or its evaluators.
+      # Supply only to select one or more case names, raising when any name is missing.
+      def cases(dataset: nil, only: nil)
+        loaded = Dataset.load(dataset || self.dataset, name: name)
+        return loaded if only.nil?
+
+        names = Array(only).map(&:to_s)
+        missing = names - loaded.map(&:name)
+        raise ArgumentError, "Unknown evaluation cases: #{missing.join(', ')}" if missing.any? || names.empty?
+
+        loaded.select { |test_case| names.include?(test_case.name) }
+      end
+
+      # Executes fresh instances for every case and repetition and returns an Evaluation::Report.
+      # A supplied dataset overrides discovery for this run. Configuration errors raise;
+      # task, assertion, and evaluator failures are recorded per case.
+      # Yields each completed Trial before starting the next. Exceptions from the block propagate.
+      # Supply id to correlate reports and instrumentation with an application record or job.
+      def run(dataset: nil, only: nil, repetitions: 1, id: nil)
+        validate_run(repetitions)
+        id = (id || SecureRandom.uuid).to_s
+        raise ArgumentError, 'An evaluation run id cannot be empty' if id.empty?
+
+        cases = self.cases(dataset:, only:)
+        groups = evaluation_groups(cases)
+        definitions = Judge::Data.copy(groups.flat_map do |backend, criteria|
+          criteria.map { |criterion| criterion.merge(evaluator: backend.description) }
+        end)
+        started_at = Time.now.utc
+        name = self.name || 'Anonymous evaluation'
+        payload = { evaluation_id: id, evaluation_name: name,
+                    total: cases.size * repetitions, completed: 0, started_at: }
+        RubyLLM.instrument('evaluation.ruby_llm', payload) do |event|
+          trials = cases.flat_map do |test_case|
+            Array.new(repetitions) do |index|
+              trial = run_trial(test_case, index + 1, groups, event)
+              yield trial if block_given?
+              trial
+            end
+          end
+          event[:report] = Report.new(name:, trials:, id:, started_at:, definitions:)
+        end
+      end
+
+      def definitions # :nodoc:
+        @definitions ||= {}
+      end
+
+      def adapters # :nodoc:
+        @adapters ||= {}
+      end
+
+      private
+
+      def run_trial(test_case, repetition, groups, run)
+        payload = run.slice(:evaluation_id, :evaluation_name, :total).merge(case: test_case.name, repetition:)
+        RubyLLM.instrument('evaluation_trial.ruby_llm', payload) do |event|
+          trial = new.run_case(test_case, repetition, groups, run_id: run[:evaluation_id])
+          event[:completed] = run[:completed] += 1
+          event[:trial] = trial
+        end
+      end
+
+      def validate_run(repetitions)
+        raise ArgumentError, 'Define perform(input) in your evaluation' if instance_method(:perform).owner == Evaluation
+        if evaluator == false && definitions.any?
+          raise ArgumentError, 'Cannot declare semantic evaluations with evaluator false'
+        end
+        return if repetitions.is_a?(Integer) && repetitions.positive?
+
+        raise ArgumentError, 'Repetitions must be a positive Integer'
+      end
+
+      def validate_criterion(name, minimum)
+        raise ArgumentError, 'An evaluation name cannot be empty' if name.to_s.empty?
+        return if minimum.nil? || (minimum.is_a?(Numeric) && minimum.finite?)
+
+        raise ArgumentError, 'Minimum must be a finite number'
+      end
+
+      def evaluation_groups(cases)
+        return {} unless evaluator
+
+        all = evaluator.question_names.to_h do |name|
+          [name, { name:, instructions: nil, minimum: nil }]
+        end.merge(definitions)
+        all[:correctness] = { name: :correctness, instructions: nil, minimum: nil } if all.empty?
+        groups = all.values.group_by { |definition| definition[:evaluator] || evaluator }
+        groups.to_h do |backend, criteria|
+          [backend, criteria.map { |criterion| prepare_criterion(backend, criterion, cases) }]
+        end
+      end
+
+      def prepare_criterion(backend, criterion, cases)
+        existing = backend.question_names.include?(criterion[:name])
+        raise ArgumentError, "Duplicate Judge question: #{criterion[:name]}" if existing && criterion[:instructions]
+        return criterion if existing || !criterion[:instructions].to_s.empty?
+
+        if criterion[:name] == :correctness && criterion[:instructions].nil?
+          return criterion.merge(instructions: correctness_instructions(cases))
+        end
+
+        raise ArgumentError, "Missing instructions: #{criterion[:name]}"
+      end
+
+      def correctness_instructions(cases)
+        missing = cases.reject(&:expected_output?).map(&:name)
+        raise ArgumentError, "Correctness requires expected_output for cases: #{missing.join(', ')}" if missing.any?
+
+        CORRECTNESS
+      end
+    end
+
+    # Returns the current case's input, reference output, and metadata, respectively.
+    attr_reader :input, :expected_output, :metadata
+    # Returns the original value returned by perform.
+    attr_reader :result
+
+    # Runs the application under evaluation. Override in your evaluation class.
+    def perform(_input)
+      raise NotImplementedError, 'Define perform(input) in your evaluation'
+    end
+
+    # Runs Ruby assertions after perform. Override to use assert, refute, and the Minitest assertion family.
+    def assertions; end
+
+    # Runs before perform on each fresh case instance. Override for application fixtures.
+    def setup; end
+
+    # Runs after each case, even when setup, perform, or an assertion fails.
+    def teardown; end
+
+    # Returns the primary answer or value extracted from result.
+    def output
+      @evidence.output
+    end
+
+    # Returns retained messages from a Chat, Agent, or Message result.
+    def messages
+      @evidence.messages
+    end
+
+    # Returns tool calls recorded in the returned conversation or message.
+    def tool_calls
+      @evidence.tool_calls
+    end
+
+    # :method: assert
+    # :call-seq:
+    #   assert(condition, message = nil)
+    #
+    # Asserts that condition is truthy. Delegates to Minitest::Assertions and
+    # records its assertion count. Requires the minitest gem, which Rails
+    # applications already include. The other Minitest assert_* and refute_*
+    # methods use the same contract. Call these from #assertions.
+
+    # :method: refute
+    # :call-seq:
+    #   refute(condition, message = nil)
+    #
+    # Asserts that condition is false or nil. See #assert.
+
+    # :method: assert_equal
+    # :call-seq:
+    #   assert_equal(expected, actual, message = nil)
+    #
+    # Asserts equality through Minitest::Assertions. See #assert.
+
+    # :method: assert_includes
+    # :call-seq:
+    #   assert_includes(collection, value, message = nil)
+    #
+    # Asserts that the collection includes value. See #assert.
+
+    # :stopdoc:
+    def method_missing(name, ...)
+      return super unless Assertions::METHOD.match?(name) && assertion_context.respond_to?(name)
+
+      assertion_context.public_send(name, ...)
+    end
+
+    def respond_to_missing?(name, include_private = nil)
+      Assertions.method?(name) || super
+    end
+
+    def run_case(test_case, repetition, groups, run_id:) # :nodoc:
+      Runner.new(self, test_case, repetition, groups, run_id:).run
+    end
+
+    def prepare(test_case) # :nodoc:
+      @input = JSON.parse(JSON.generate(test_case.inputs))
+      @expected_output = test_case.expected_output
+      @metadata = test_case.metadata
+    end
+
+    def execute # :nodoc:
+      @result = perform(input)
+      @evidence = Evidence.new(result, adapters: self.class.adapters)
+    end
+
+    def assertion_context # :nodoc:
+      @assertion_context ||= Assertions.new
+    end
+
+    def assertion_count # :nodoc:
+      @assertion_context&.assertions.to_i
+    end
+    # :startdoc:
+  end
+end

data/lib/ruby_llm/image.rb

@@ -43,6 +43,8 @@ module RubyLLM
     # request vocabulary and merges them into the request as-is.
     # +context:+ supplies a Context whose configuration replaces the
     # global one. +metadata:+ is included in the instrumentation payload.
+    # +owner:+ attributes the usage to a record, such as a user, and wins
+    # over RubyLLM.with_usage_owner.
     #
     #   image = RubyLLM.paint("A small watercolor robot", model: "gpt-image-2")
     #
@@ -67,7 +69,8 @@ module RubyLLM
                    with: nil,
                    mask: nil,
                    provider_options: {},
-                   metadata: nil)
+                   metadata: nil,
+                   owner: nil)
       config = context&.config || RubyLLM.config
       model ||= config.default_image_model
       model, provider_instance = Models.resolve(model, provider: provider, assume_model_exists: assume_model_exists,
@@ -88,7 +91,9 @@ module RubyLLM
       }
 
       RubyLLM.instrument('image.ruby_llm', payload, config: config) do |event|
-        result = provider_instance.paint(prompt, model:, size:, count:, with:, mask:, provider_options:)
+        result = Accounting::Usage.owned_by(owner) do
+          provider_instance.paint(prompt, model:, size:, count:, with:, mask:, provider_options:)
+        end
         images = Support::Utils.to_safe_array(result)
         event[:result] = result
         event[:response_model] = images.first&.model
@@ -174,8 +179,11 @@ module RubyLLM
       Cost.new(tokens:, model: model_info, category: :images, input_details: input_tokens_details)
     end
 
-    # Returns the registry Model for #model, or +nil+ if the model id
-    # is missing or not in the registry.
+    attr_writer :model_info # :nodoc:
+
+    # Returns the image's Model from its provider's registry, falling back
+    # to the requested model when the response ID is unknown. Images built
+    # without a request look up #model, or return +nil+ if unknown.
     def model_info
       return unless model
 

data/lib/ruby_llm/judge/builder.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class Judge
+    class Builder # :nodoc:
+      def self.resolve(value, scope:)
+        return value unless value.is_a?(::Proc)
+
+        new(scope).evaluate(value)
+      end
+
+      def initialize(scope)
+        @scope = scope
+        @data = {}
+      end
+
+      def evaluate(block)
+        result = block.arity.zero? ? instance_exec(&block) : instance_exec(self, &block)
+        return result if @data.empty?
+
+        unless result.equal?(self) || result.nil?
+          ::Kernel.raise ::ArgumentError, 'A judgment block must declare fields or return a value, not both'
+        end
+
+        @data
+      end
+
+      def method_missing(name, *values, &block)
+        return @scope.public_send(name) if runtime_input?(name, values, block)
+
+        if values.size > 1 || (!values.empty? && block)
+          ::Kernel.raise ::ArgumentError, "#{name} accepts one value or a block"
+        end
+        ::Kernel.raise ::ArgumentError, "Duplicate judgment field: #{name}" if @data.key?(name)
+
+        @data[name] = block ? ::RubyLLM::Judge::Builder.resolve(block, scope: @scope) : values.first
+        self
+      end
+
+      def respond_to_missing?(_name, _include_private = nil)
+        true
+      end
+
+      private
+
+      def runtime_input?(name, values, block)
+        values.empty? && !block && @scope.respond_to?(name)
+      end
+    end
+  end
+end

data/lib/ruby_llm/judge/data.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class Judge
+    module Data # :nodoc:
+      module_function
+
+      def copy(value, &resolver)
+        case value
+        when Proc then resolver ? copy(resolver.call(value), &resolver) : value
+        when Hash then copy_hash(value, &resolver)
+        when Array then value.map { |item| copy(item, &resolver) }.freeze
+        when String then value.dup.freeze
+        when Symbol then value.to_s.freeze
+        when Float
+          raise ArgumentError, 'Judgment data must contain finite numbers' unless value.finite?
+
+          value
+        when Integer, TrueClass, FalseClass, NilClass then value
+        else raise ArgumentError, "Unsupported judgment data: #{value.class}; pass JSON-compatible values"
+        end
+      end
+
+      def copy_hash(value, &resolver)
+        seen = {}
+        value.each_with_object({}) do |(key, item), result|
+          name = key.to_s
+          unless key.is_a?(String) || key.is_a?(Symbol) || key == true || key == false
+            raise ArgumentError, 'Judgment data keys must be Strings, Symbols, or booleans'
+          end
+          raise ArgumentError, "Duplicate judgment key: #{name}" if seen[name]
+
+          seen[name] = true
+          result[key.is_a?(String) ? key.dup.freeze : key] = copy(item, &resolver)
+        end.freeze
+      end
+
+      def description?(value)
+        value.nil? || value.is_a?(String) || value.is_a?(Hash) || value.is_a?(Array)
+      end
+    end
+  end
+end

data/lib/ruby_llm/judge/question.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class Judge
+    class Question # :nodoc:
+      CRITERIA_KEYS = { probability: :criteria, choice: :options, score: :levels }.freeze
+
+      attr_reader :name, :type, :instructions, :criteria
+
+      def initialize(name, type:, instructions: nil, criteria: nil, &block)
+        validate_name!(name)
+        raise ArgumentError, "Unknown judgment type: #{type.inspect}" unless CRITERIA_KEYS.key?(type)
+        raise ArgumentError, 'Pass question criteria or a block, not both' if !criteria.nil? && block
+
+        @name = name.is_a?(String) ? name.dup.freeze : name
+        @type = type
+        @instructions = Data.copy(instructions)
+        @criteria = Data.copy(block || criteria)
+        freeze
+      end
+
+      def self.from_h(name, definition)
+        raise ArgumentError, 'Each question must be a Hash' unless definition.is_a?(Hash)
+
+        definition = definition.transform_keys(&:to_sym)
+        type = definition[:type]
+        type = type.to_sym if type.is_a?(String)
+        key = CRITERIA_KEYS.fetch(type) { raise ArgumentError, "Unknown judgment type: #{type.inspect}" }
+        extra = definition.keys - [:type, :instructions, key]
+        raise ArgumentError, "Unknown question options: #{extra.join(', ')}" unless extra.empty?
+
+        new(name, type:, instructions: definition[:instructions], criteria: definition[key])
+      end
+
+      def resolve(scope)
+        resolver = ->(value) { Builder.resolve(value, scope:) }
+        resolved = self.class.new(name, type:, instructions: Data.copy(instructions, &resolver),
+                                        criteria: Data.copy(criteria, &resolver))
+        resolved.validate!
+        resolved
+      end
+
+      def validate!
+        unless Data.description?(instructions)
+          raise ArgumentError,
+                'Question instructions must be text, a Hash, an Array, or nil'
+        end
+
+        case type
+        when :probability then validate_probability!
+        when :choice then validate_choice!
+        when :score then validate_score!
+        end
+      end
+
+      private
+
+      def validate_name!(name)
+        unless name.is_a?(String) || name.is_a?(Symbol)
+          raise ArgumentError, 'A question name must be a String or Symbol'
+        end
+        raise ArgumentError, 'A question name cannot be empty' if name.to_s.empty?
+      end
+
+      def validate_probability!
+        return if criteria.nil?
+
+        unless criteria.is_a?(Hash) && (criteria.keys.map(&:to_s) - %w[yes no true false]).empty?
+          raise ArgumentError, 'Probability criteria must describe yes and no'
+        end
+
+        positive = %w[yes true]
+        aliases = criteria.keys.map { |key| positive.include?(key.to_s) }
+        raise ArgumentError, 'Probability criteria contain duplicate outcomes' unless aliases.uniq.size == aliases.size
+
+        validate_descriptions!(criteria.values)
+      end
+
+      def validate_choice!
+        raise ArgumentError, 'A choice needs a nonempty Hash of options' unless criteria.is_a?(Hash) && !criteria.empty?
+        unless criteria.keys.all? { |key| (key.is_a?(String) || key.is_a?(Symbol)) && !key.to_s.empty? }
+          raise ArgumentError, 'Choice options must have nonempty String or Symbol names'
+        end
+
+        validate_descriptions!(criteria.values)
+      end
+
+      def validate_score!
+        raise ArgumentError, 'A score needs a nonempty Array of levels' unless criteria.is_a?(Array) && !criteria.empty?
+
+        validate_descriptions!(criteria)
+      end
+
+      def validate_descriptions!(values)
+        return if values.all? { |value| Data.description?(value) }
+
+        raise ArgumentError, 'Descriptions must be text, a Hash, an Array, or nil'
+      end
+    end
+  end
+end