ruby_llm 1.15.0 → 2.0.0.rc1

data/lib/ruby_llm/image.rb

@@ -1,82 +1,205 @@
 # frozen_string_literal: true
 
+require 'base64'
+
 module RubyLLM
-  # Represents a generated image from an AI model.
+  # An Image is a generated or edited image. Save it to a file with #save
+  # or read its bytes with #to_blob. Both handle hosted URLs and inline data.
+  #
+  #   image = RubyLLM.paint("a sunset over mountains in watercolor style")
+  #   image.save("sunset.png")
+  #
   class Image
-    attr_reader :url, :data, :mime_type, :revised_prompt, :model_id, :usage
+    include Support::Inspectable
+    include Accounting::Usage::Result
+
+    # The URL of the hosted image, for providers that return one, or +nil+.
+    attr_reader :url
+
+    # The Base64-encoded image data, for providers that return the image
+    # inline, or +nil+.
+    attr_reader :data
+
+    # The MIME type of the image data, such as <tt>"image/png"</tt>.
+    attr_reader :mime_type
+
+    # The provider's rewritten version of the prompt, when reported.
+    attr_reader :revised_prompt
+
+    # The id of the model that generated the image.
+    attr_reader :model
+
+    # Generates an image from +prompt+ and returns an Image. Most code
+    # calls this through RubyLLM.paint.
+    #
+    # +model:+ selects the image model and defaults to the configured
+    # +default_image_model+. +provider:+ forces a specific provider, and
+    # +assume_model_exists:+ skips the registry lookup, which is useful
+    # for custom endpoints. +size:+ requests dimensions on models that
+    # support it. +count:+ asks for several images in one request, returning
+    # an Array of Images instead of one. +with:+ passes one or more source
+    # images for editing, and +mask:+ constrains which parts of the image
+    # may change. +provider_options:+ takes options in the provider's
+    # request vocabulary and merges them into the request as-is.
+    # +context:+ supplies a Context whose configuration replaces the
+    # global one. +metadata:+ is included in the instrumentation payload.
+    #
+    #   image = RubyLLM.paint("A small watercolor robot", model: "gpt-image-2")
+    #
+    #   images = RubyLLM.paint("A small watercolor robot", count: 4)
+    #   images.each_with_index { |image, i| image.save("robot-#{i}.png") }
+    #
+    #   RubyLLM.paint(
+    #     "Turn the logo green and keep the background transparent",
+    #     model: "gpt-image-2",
+    #     with: "logo.png"
+    #   )
+    #
+    # Providers that cannot generate several images in one request ignore
+    # +count:+ and return a single Image.
+    def self.paint(prompt,
+                   model: nil,
+                   provider: nil,
+                   assume_model_exists: false,
+                   size: nil,
+                   count: nil,
+                   context: nil,
+                   with: nil,
+                   mask: nil,
+                   provider_options: {},
+                   metadata: nil)
+      config = context&.config || RubyLLM.config
+      model ||= config.default_image_model
+      model, provider_instance = Models.resolve(model, provider: provider, assume_model_exists: assume_model_exists,
+                                                       config: config)
+      empty_tokens = Tokens.new
+      payload = {
+        provider: provider_instance.slug,
+        provider_class: provider_instance.class.display_name,
+        model: model.id,
+        model_info: model,
+        prompt: prompt,
+        size: size,
+        count: count,
+        provider_options: provider_options,
+        metadata: metadata,
+        tokens: empty_tokens,
+        cost: Cost.new(tokens: empty_tokens, model:, category: :images)
+      }
+
+      RubyLLM.instrument('image.ruby_llm', payload, config: config) do |event|
+        result = provider_instance.paint(prompt, model:, size:, count:, with:, mask:, provider_options:)
+        images = Support::Utils.to_safe_array(result)
+        event[:result] = result
+        event[:response_model] = images.first&.model
+        event[:tokens] = Tokens.aggregate(images.map(&:tokens))
+        event[:cost] = Cost.aggregate(images.map(&:cost))
+        result
+      end
+    end
 
-    def initialize(url: nil, data: nil, mime_type: nil, revised_prompt: nil, model_id: nil, usage: {}) # rubocop:disable Metrics/ParameterLists
+    # :stopdoc:
+
+    # Set by the protocol that generated the image, so a Context's
+    # connection settings reach #to_blob.
+    attr_writer :config
+
+    def config
+      @config || RubyLLM.config
+    end
+
+    def initialize(url: nil, data: nil, mime_type: nil, revised_prompt: nil, model: nil, usage: {})
       @url = url
       @data = data
       @mime_type = mime_type
       @revised_prompt = revised_prompt
-      @model_id = model_id
-      @usage = usage
+      @model = model
+      @raw_usage = usage
     end
+    # :startdoc:
 
+    # Returns +true+ if the image holds inline Base64 data, +false+ otherwise.
     def base64?
       !@data.nil?
     end
 
+    # Returns the raw binary image bytes, decoding #data when present or
+    # downloading from #url otherwise.
+    #
+    #   image_bytes = image.to_blob
+    #
     def to_blob
       if base64?
         Base64.decode64 @data
       else
-        response = Connection.basic.get @url
+        response = Transport::Connection.basic(config).get @url
         response.body
       end
     end
 
+    # Writes the binary image to +path+, expanding it first. Returns
+    # +path+ as given.
+    #
+    #   image.save("steampunk_owl.png")
+    #
     def save(path)
       File.binwrite(File.expand_path(path), to_blob)
       path
     end
 
-    def self.paint(prompt, # rubocop:disable Metrics/ParameterLists
-                   model: nil,
-                   provider: nil,
-                   assume_model_exists: false,
-                   size: '1024x1024',
-                   context: nil,
-                   with: nil,
-                   mask: nil,
-                   params: {})
-      config = context&.config || RubyLLM.config
-      model ||= config.default_image_model
-      model, provider_instance = Models.resolve(model, provider: provider, assume_exists: assume_model_exists,
-                                                       config: config)
-      model_id = model.id
-
-      provider_instance.paint(prompt, model: model_id, size:, with:, mask:, params:)
-    end
-
+    # Returns a Tokens with usage across every provider attempt.
+    # Its fields are +nil+ when none were reported.
+    #
+    #   image.tokens.input
+    #   image.tokens.output
+    #
     def tokens
-      @tokens ||= Tokens.build(
-        input: usage_value('input_tokens'),
-        output: usage_value('output_tokens')
+      return ruby_llm_usage_tokens unless ruby_llm_usage_entries.empty?
+
+      @tokens ||= Tokens.new(
+        input: raw_usage['input_tokens'],
+        output: raw_usage['output_tokens'],
+        reported_cost: raw_usage['cost']
       )
     end
 
+    # Returns a Cost across every provider attempt, using reported prices
+    # when available and registry pricing otherwise.
+    #
+    #   image.cost.total
+    #
     def cost
+      return ruby_llm_usage_cost unless ruby_llm_usage_entries.empty?
+
       Cost.new(tokens:, model: model_info, category: :images, input_details: input_tokens_details)
     end
 
+    # Returns the registry Model for #model, or +nil+ if the model id
+    # is missing or not in the registry.
     def model_info
-      return unless model_id
+      return unless model
 
-      @model_info ||= RubyLLM.models.find(model_id)
+      @model_info ||= RubyLLM.models.find(model)
     rescue ModelNotFoundError
       nil
     end
 
     private
 
+    attr_reader :raw_usage
+
     def input_tokens_details
-      usage_value('input_tokens_details')
+      raw_usage['input_tokens_details']
     end
 
-    def usage_value(key)
-      usage[key] || usage[key.to_sym]
+    def inspect_attributes # :nodoc:
+      {
+        model: model,
+        mime_type: mime_type,
+        url: url,
+        data: data && "#{data.bytesize} bytes",
+        revised_prompt: revised_prompt
+      }
     end
   end
 end

data/lib/ruby_llm/message.rb

@@ -1,127 +1,306 @@
 # frozen_string_literal: true
 
 module RubyLLM
-  # A single message in a chat conversation.
+  # A Message is a single entry in a chat conversation: a user prompt, an
+  # assistant reply, a system instruction, or a tool result. Chat#ask
+  # returns the model's reply as a Message, and Chat#messages holds the
+  # transcript as an array of them.
+  #
+  #   response = chat.ask "What is the capital of France?"
+  #   response.role          # => :assistant
+  #   response.content       # => "The capital of France is Paris."
+  #   response.finish_reason # => :stop
+  #
+  # A Message also carries everything else the provider returned: token
+  # usage (#tokens), reasoning output (#thinking), source citations
+  # (#citations), and requested tool calls (#tool_calls).
   class Message
+    include Support::Inspectable
+    include Accounting::Usage::Result
+
+    # The valid message roles: +:system+, +:user+, +:assistant+, and +:tool+.
     ROLES = %i[system user assistant tool].freeze
 
-    attr_reader :role, :model_id, :tool_calls, :tool_call_id, :raw, :thinking, :tokens
-    attr_writer :content
+    # The role of the message: +:system+, +:user+, +:assistant+, or +:tool+.
+    attr_reader :role
+
+    # The message text as a String. Empty for assistant messages that only
+    # request tool calls.
+    attr_reader :content
+
+    # The files sent or returned with the message, as an array of
+    # Attachment objects.
+    attr_reader :attachments
+
+    # The ID of the model that produced the message, +nil+ on user messages.
+    attr_reader :model
+
+    # The tool calls the assistant requested, as a Hash of ToolCall objects
+    # keyed by call ID, or +nil+.
+    attr_reader :tool_calls
+
+    # The ID of the tool call this message answers. Set only on tool result
+    # messages.
+    attr_reader :tool_call_id
+
+    # The raw provider response: a Faraday::Response, or the result body
+    # Hash for messages retrieved from a Batch.
+    attr_reader :raw
+
+    # The model's reasoning output as a Thinking object, or +nil+ when the
+    # provider returned none.
+    attr_reader :thinking
+
+    # The source citations as an array of Citation objects, normalized
+    # across providers.
+    attr_reader :citations
+
+    # Why the model stopped: +:stop+, +:max_tokens+, +:tool_calls+, or
+    # +:content_filter+. Any other reason comes through as the provider
+    # spelled it, such as Anthropic's +:pause_turn+.
+    attr_reader :finish_reason
+
+    # The provider-executed tool steps in this response, as an array of
+    # ServerToolCall objects. Empty unless the chat enabled tools with
+    # Chat#with_server_tools and the model used one.
+    attr_reader :server_tool_calls
+
+    # The provider-shaped content blocks of this assistant message, kept
+    # verbatim when the response used server tools so later requests can
+    # replay the turn exactly. +nil+ otherwise.
+    attr_reader :raw_content # :nodoc:
 
-    def initialize(options = {})
+    # The provider-shaped reasoning payload of this assistant message, kept
+    # verbatim so later requests can replay the model's reasoning exactly.
+    # +nil+ when the provider returned none.
+    attr_reader :raw_reasoning # :nodoc:
+
+    # The Chat this message belongs to, set when it is added to a
+    # conversation. Backs #tool_results.
+    attr_accessor :conversation # :nodoc:
+
+    def initialize(options = {}) # :nodoc:
       @role = options.fetch(:role).to_sym
-      @tool_calls = options[:tool_calls]
-      @content = normalize_content(options.fetch(:content), role: @role, tool_calls: @tool_calls)
-      @model_id = options[:model_id]
+      @tool_calls = coerce_tool_calls(options[:tool_calls])
+      @content = normalize_content(options.fetch(:content))
+      @config = options[:config]
+      @attachments = Attachment.wrap(options[:attachments], config: @config)
+      @model = options[:model]
+      @supplied_cost = coerce_value(options[:cost], Cost)
       @tool_call_id = options[:tool_call_id]
-      @tokens = options[:tokens] || Tokens.build(
+      @tokens = options[:tokens] || Tokens.new(
         input: options[:input_tokens],
         output: options[:output_tokens],
-        cached: options[:cached_tokens],
-        cache_creation: options[:cache_creation_tokens],
+        cache_read: options[:cache_read_tokens],
+        cache_write: options[:cache_write_tokens],
         thinking: options[:thinking_tokens],
-        reasoning: options[:reasoning_tokens]
+        server_tool_use: options[:server_tool_use],
+        reported_cost: options[:reported_cost]
       )
       @raw = options[:raw]
-      @thinking = options[:thinking]
+      @thinking = coerce_thinking(options[:thinking], options[:thinking_signature])
+      @citations = Array(options[:citations]).map { |citation| coerce_value(citation, Citation) }
+      @server_tool_calls = Array(options[:server_tool_calls]).map { |call| coerce_value(call, ServerToolCall) }
+      @raw_content = options[:raw_content]
+      @raw_reasoning = options[:raw_reasoning]
+      @finish_reason = options[:finish_reason]&.to_sym
+      self.ruby_llm_usage_entries = options[:usage_entries] if options[:usage_entries]
+      @cache_until_here = options.fetch(:cache_until_here, false)
 
       ensure_valid_role
     end
 
-    def content
-      if @content.is_a?(Content) && @content.text && @content.attachments.empty?
-        @content.text
-      else
-        @content
-      end
+    # Returns #content parsed as JSON, memoized after the first call.
+    # Useful for reading structured output responses.
+    #
+    #   response = chat.with_schema(PersonSchema).ask "Generate a person"
+    #   response.parsed # => {"name" => "Alice", "age" => 30}
+    #
+    def parsed
+      return if content.nil? || content.empty?
+
+      @parsed ||= JSON.parse(content)
+    end
+
+    def with_attachments(attachments) # :nodoc:
+      wrapped = Attachment.wrap(attachments, config: @config)
+      dup.tap { |message| message.instance_variable_set(:@attachments, wrapped) }
     end
 
+    # Returns +true+ if the assistant requested one or more tool calls,
+    # +false+ otherwise.
     def tool_call?
       !tool_calls.nil? && !tool_calls.empty?
     end
 
+    # Returns +true+ if the message carries the result of a tool call,
+    # +false+ otherwise.
     def tool_result?
       !tool_call_id.nil? && !tool_call_id.empty?
     end
 
+    # Returns the tool result messages answering this message's tool calls,
+    # or an empty array when it made none. Mirrors the +tool_results+
+    # association on acts_as_message records.
     def tool_results
-      content if tool_result?
-    end
+      return [] unless tool_call? && conversation
 
-    def input_tokens
-      tokens&.input
+      conversation.messages.select do |message|
+        message.tool_result? && tool_calls.key?(message.tool_call_id)
+      end
     end
 
-    def output_tokens
-      tokens&.output
+    # Returns +true+ if #finish_reason indicates the model finished
+    # normally, +false+ otherwise. A turn that stopped to call tools is
+    # reported by #tool_call_stop? instead, whatever the provider named it.
+    def stopped?
+      finish_reason == :stop && !tool_call?
     end
 
-    def cached_tokens
-      tokens&.cached
+    # Returns +true+ if the response was cut off by a token limit,
+    # +false+ otherwise.
+    def max_tokens?
+      finish_reason == :max_tokens
     end
 
-    def cache_creation_tokens
-      tokens&.cache_creation
+    # Returns +true+ if the model stopped to request tool calls,
+    # +false+ otherwise.
+    def tool_call_stop?
+      finish_reason == :tool_calls || (tool_call? && finish_reason == :stop)
     end
 
-    def cache_read_tokens
-      tokens&.cache_read
+    # Returns +true+ if a provider safety filter stopped the response,
+    # +false+ otherwise.
+    def content_filtered?
+      finish_reason == :content_filter
     end
 
-    def cache_write_tokens
-      tokens&.cache_write
+    # Returns usage aggregated across every provider attempt that produced this
+    # message. Messages constructed by hand report the token counts they were
+    # built with.
+    def tokens
+      return @tokens if ruby_llm_usage_entries.empty?
+
+      ruby_llm_usage_tokens
     end
 
-    def thinking_tokens
-      tokens&.thinking
+    # Returns a Cost pricing this message's token usage in US dollars.
+    # Uses recorded attempt costs, an explicitly supplied +cost:+, or pricing
+    # from #model_info. An explicit +model:+ overrides those costs for repricing.
+    #
+    #   response.cost.total
+    #
+    def cost(model: nil)
+      return ruby_llm_usage_cost if model.nil? && ruby_llm_usage_entries.any?
+      return @supplied_cost if model.nil? && @supplied_cost
+
+      Cost.new(tokens:, model: model || model_info)
     end
 
-    def reasoning_tokens
-      tokens&.thinking
+    # Marks this message as an explicit prompt cache boundary. Providers
+    # with boundary controls use the conversation up to and including this
+    # message as the cacheable prefix. Returns +self+.
+    #
+    #   chat.add_message(role: :user, content: long_context).cache_until_here
+    #
+    def cache_until_here
+      @cache_until_here = true
+      self
     end
 
-    def cost(model: nil)
-      Cost.new(tokens:, model: model || model_info)
+    # Returns +true+ if the message carries an explicit prompt cache
+    # boundary, +false+ otherwise.
+    def cache_until_here?
+      @cache_until_here
     end
 
+    # Returns a Hash of the message's attributes, with token counts merged
+    # in as +:input_tokens+, +:output_tokens+, and related keys. Omits
+    # +nil+ values and empty attachment and citation lists. Includes +:cost+
+    # only when supplied explicitly, preserving unknown costs on round-trip.
     def to_h
       {
         role: role,
         content: content,
-        model_id: model_id,
-        tool_calls: tool_calls,
+        attachments: list_to_h(attachments),
+        model: model,
+        cost: @supplied_cost && cost.to_h,
+        tool_calls: tool_calls&.transform_values(&:to_h),
         tool_call_id: tool_call_id,
         thinking: thinking&.text,
-        thinking_signature: thinking&.signature
-      }.merge(tokens ? tokens.to_h : {}).compact
-    end
-
-    def instance_variables
-      super - [:@raw]
+        thinking_signature: thinking&.signature,
+        citations: list_to_h(citations),
+        server_tool_calls: list_to_h(server_tool_calls),
+        raw_content: raw_content,
+        raw_reasoning: raw_reasoning,
+        finish_reason: finish_reason,
+        cache_until_here: cache_until_here? || nil
+      }.merge(tokens.to_h).compact
     end
 
+    # Returns the Model record for #model from the model registry, or
+    # +nil+ when the message has no model or the model is unknown.
     def model_info
-      return unless model_id
+      return unless model
 
-      @model_info ||= RubyLLM.models.find(model_id)
+      @model_info ||= RubyLLM.models.find(model)
     rescue ModelNotFoundError
       nil
     end
 
     private
 
-    def normalize_content(content, role:, tool_calls:)
-      return '' if role == :assistant && content.nil? && tool_calls && !tool_calls.empty?
+    def list_to_h(list)
+      list.empty? ? nil : list.map(&:to_h)
+    end
 
-      case content
-      when String then Content.new(content)
-      when Hash then Content.new(content[:text], content)
-      else content
+    def coerce_tool_calls(tool_calls)
+      return tool_calls unless tool_calls.is_a?(Hash)
+
+      tool_calls.to_h do |id, call|
+        next [id, call] unless call.is_a?(Hash)
+
+        attributes = call.transform_keys(&:to_sym)
+        [id, ToolCall.new(id: attributes[:id] || id, name: attributes[:name],
+                          arguments: attributes[:arguments] || {},
+                          thought_signature: attributes[:thought_signature], remote: attributes.fetch(:remote, false))]
+      end
+    end
+
+    def coerce_thinking(thinking, signature)
+      case thinking
+      when nil, Thinking then thinking
+      when Hash then Thinking.build(**thinking.transform_keys(&:to_sym).slice(:text, :signature))
+      else Thinking.build(text: thinking.to_s, signature: signature)
       end
     end
 
+    def coerce_value(value, klass)
+      value.is_a?(Hash) ? klass.from_h(value) : value
+    end
+
+    def normalize_content(content)
+      return '' if role == :assistant && content.nil? && tool_calls && !tool_calls.empty?
+      return content if content.nil? || content.is_a?(String)
+
+      raise ArgumentError,
+            "Message content must be a String, got #{content.class}. " \
+            'Pass files via attachments: and structured data as JSON.'
+    end
+
     def ensure_valid_role
       raise InvalidRoleError, "Expected role to be one of: #{ROLES.join(', ')}" unless ROLES.include?(role)
     end
+
+    def inspect_attributes # :nodoc:
+      {
+        role: role,
+        content: content,
+        tool_calls: tool_calls&.values&.map(&:name),
+        tool_call_id: tool_call_id,
+        model: model,
+        finish_reason: finish_reason
+      }
+    end
   end
 end

data/lib/ruby_llm/model/modalities.rb

@@ -1,16 +1,29 @@
 # frozen_string_literal: true
 
 module RubyLLM
-  module Model
-    # Holds and manages input and output modalities for a language model
+  class Model
+    # A Model::Modalities lists the kinds of content a model accepts and
+    # produces, as arrays of Strings. Instances come from Model#modalities.
+    #
+    #   model = RubyLLM.models.find('gpt-5.6')
+    #   model.modalities.input   # => ["text", "image", "pdf"]
+    #   model.modalities.output  # => ["text"]
+    #
     class Modalities
-      attr_reader :input, :output
+      # The input modalities as an array of Strings,
+      # e.g. <tt>["text", "image", "pdf"]</tt>.
+      attr_reader :input
 
-      def initialize(data)
+      # The output modalities as an array of Strings,
+      # e.g. <tt>["text"]</tt> or <tt>["embeddings"]</tt>.
+      attr_reader :output
+
+      def initialize(data) # :nodoc:
         @input = Array(data[:input]).map(&:to_s)
         @output = Array(data[:output]).map(&:to_s)
       end
 
+      # Returns the modalities as a Hash with +:input+ and +:output+ keys.
       def to_h
         {
           input: input,