llm.rb 12.1.0 → 12.3.0

data/lib/llm/context.rb

@@ -92,6 +92,8 @@ module LLM
       @params[:tools] = tools unless tools.empty?
       @params[:store] ||= false if @mode == :responses
       @messages = LLM::Buffer.new(llm)
+      extra = @params.slice(:model, :tools).merge!(ctx: self, tracer:)
+      @params[:stream] = LLM::Stream.try(@params[:stream], extra:)
     end
 
     ##
@@ -310,10 +312,7 @@ module LLM
     #  A list of functions to exclude from the wait
     # @return [Array<LLM::Function::Return>]
     def wait(strategy, except: [])
-      if LLM::Stream === stream && !stream.queue.empty?
-        @queue = stream.queue
-        @queue.wait
-      else
+      if stream.queue.empty?
         tools  = except.empty? ? functions : functions - except
         guards = guarded_returns(tools:)
         return guards if guards
@@ -321,6 +320,9 @@ module LLM
         returns = @queue.wait
         emit_tool_returns(tools, returns)
         returns
+      else
+        @queue = stream.queue
+        @queue.wait
       end
     ensure
       @queue = nil
@@ -426,7 +428,7 @@ module LLM
     #  A tracer, or nil.
     # @return [void]
     def tracer=(other)
-      @llm.tracer = nil
+      @llm.tracer = other || LLM::Tracer::Null.new(@llm)
     end
 
     ##
@@ -502,25 +504,11 @@ module LLM
 
     private
 
-    ##
-    # Binds runtime metadata onto an active stream.
-    # @api private
-    def bind!(stream, model, tools)
-      return unless LLM::Stream === stream
-      @stream = stream
-      stream.extra[:ctx] = self
-      stream.extra[:tracer] = tracer
-      stream.extra[:model] = model
-      stream.extra[:tools] = tools
-    end
-
     ##
     # Returns the bound stream queue, if available.
     # @api private
     def queue
-      [@queue, stream&.queue].compact.first
-    rescue NoMethodError
-      nil
+      [@queue, stream.queue].compact.first
     end
 
     ##
@@ -543,12 +531,13 @@ module LLM
     # Rewrites a prompt and params through the configured transformer.
     # @api private
     def transform(prompt, params)
+      transformer = self.transformer
       return [prompt, params] unless transformer
       stream = params[:stream]
-      stream.on_transform(self, transformer) if LLM::Stream === stream
+      stream.on_transform(self, transformer)
       transformer.call(self, prompt, params)
     ensure
-      stream.on_transform_finish(self, transformer) if LLM::Stream === stream
+      stream.on_transform_finish(self, transformer) if transformer
     end
 
     ##
@@ -556,8 +545,10 @@ module LLM
     # @api private
     def respond(prompt, params)
       params = @params.merge(params)
+      extra = params.slice(:model, :tools).merge!(ctx: self, tracer:)
+      params[:stream] = LLM::Stream.try(params[:stream], extra:)
       prompt, params = transform(prompt, params)
-      bind!(params[:stream], params[:model], params[:tools])
+      @stream = params[:stream]
       res_id = params[:store] == false ? nil : @messages.find(&:assistant?)&.response&.response_id
       input = res_id ? [] : @messages.to_a
       params = params.merge(previous_response_id: res_id, input:).compact
@@ -570,8 +561,10 @@ module LLM
     def complete(prompt, params)
       params = params.merge(messages: @messages.to_a)
       params = @params.merge(params)
+      extra = params.slice(:model, :tools).merge!(ctx: self, tracer:)
+      params[:stream] = LLM::Stream.try(params[:stream], extra:)
       prompt, params = transform(prompt, params)
-      bind!(params[:stream], params[:model], params[:tools])
+      @stream = params[:stream]
       [prompt, params, @llm.complete(prompt, params)]
     end
 
@@ -590,7 +583,6 @@ module LLM
     # Emits tool return callbacks for directly waited function work.
     # @api private
     def emit_tool_returns(tools, returns)
-      return unless LLM::Stream === stream
       returns.each_with_index { |result, index| stream.on_tool_return(tools[index], result) }
     end
 
@@ -604,7 +596,7 @@ module LLM
     def repair!(messages, prompt)
       message = messages.last
       return unless message&.tool_call?
-      returns = self.returns + [*prompt].grep(LLM::Function::Return)
+      returns = [self.returns, prompt].flatten.grep(LLM::Function::Return)
       cancelled = []
       [*message.extra.tool_calls].each do |tool|
         next if returns.any? { _1.id == tool[:id] }

data/lib/llm/function.rb

@@ -390,6 +390,12 @@ class LLM::Function
         type: "function", name: @name, description: @description,
         parameters: (@params || {type: "object", properties: {}}).to_h.merge(additionalProperties: false), strict: false
       }.compact
+    when "LLM::Mistral"
+      params = @params || {type: "object", properties: {}}
+      {
+        type: "function",
+        function: {name: @name, description: @description, parameters: params}
+      }.compact
     else
       params = @params || {type: "object", properties: {}}
       {

data/lib/llm/object.rb

@@ -148,6 +148,19 @@ class LLM::Object < BasicObject
     SINGLETON.from @h.merge(other)
   end
 
+  ##
+  # @param [Hash, #to_h] other
+  #  The hash to merge in place
+  # @return [LLM::Object]
+  #  Returns self
+  def merge!(other = UNDEFINED)
+    return SINGLETON.get(@h, :merge!) if other.equal?(UNDEFINED)
+    other = ::Hash.try_convert(other)
+    raise TypeError, "#{other} cannot be coerced into a Hash" unless other
+    @h.merge!(other)
+    self
+  end
+
   ##
   # @param [#to_s, #to_sym] k
   #  The key name

data/lib/llm/provider.rb

@@ -26,7 +26,7 @@ class LLM::Provider
   #  Requires the net-http-persistent gem.
   # @param [LLM::Transport, Class, nil] transport
   #  Optional override with any {LLM::Transport} instance or subclass.
-  def initialize(key:, host:, port: 443, timeout: 60, ssl: true, base_path: "", persistent: false, transport: nil)
+  def initialize(key:, host:, port: 443, timeout: 180, ssl: true, base_path: "", persistent: false, transport: nil)
     @key = key
     @host = host
     @port = port
@@ -71,6 +71,16 @@ class LLM::Provider
     raise NotImplementedError
   end
 
+  ##
+  # @note
+  #  This feature is not implemented by all providers,
+  #  and it will raise NotImplementedError for providers
+  #  that do not support it.
+  # @return [LLM::Response]
+  def ocr(...)
+    raise NotImplementedError
+  end
+
   ##
   # Provides an interface to the chat completions API
   # @example
@@ -303,7 +313,7 @@ class LLM::Provider
   def with_tracer(tracer)
     had_override = weakmap.key?(self)
     previous = weakmap[self]
-    weakmap[self] = tracer
+    weakmap[self] = tracer || LLM::Tracer::Null.new(self)
     yield
   ensure
     if had_override
@@ -332,13 +342,6 @@ class LLM::Provider
     transport.request_owner
   end
 
-  ##
-  # @param [Object] stream
-  # @return [Boolean]
-  def streamable?(stream)
-    LLM::Stream === stream || stream.respond_to?(:<<)
-  end
-
   ##
   # @return [Boolean]
   #  Returns true when an API key is configured

data/lib/llm/providers/anthropic.rb

@@ -92,10 +92,10 @@ module LLM
 
     ##
     # Returns the default model for chat completions
-    # @see https://docs.anthropic.com/en/docs/about-claude/models/all-models#model-comparison-table claude-sonnet-4-20250514
+    # @see https://docs.anthropic.com/en/docs/about-claude/models/all-models#model-comparison-table claude-opus-4-8
     # @return [String]
     def default_model
-      "claude-sonnet-4-20250514"
+      "claude-opus-4-8"
     end
 
     ##
@@ -151,9 +151,8 @@ module LLM
       params = {role: :user, model: default_model, max_tokens: 1024}.merge!(params)
       tools = resolve_tools(params.delete(:tools))
       params = [params, adapt_tools(tools)].inject({}, &:merge!).compact
-      role, stream = params.delete(:role), params.delete(:stream)
-      params[:stream] = true if streamable?(stream) || stream == true
-      [params, stream, tools, role]
+      role, stream = params.delete(:role), LLM::Stream.try(params.delete(:stream))
+      [params.merge!(stream: stream.enabled?), stream, tools, role]
     end
 
     def build_complete_request(prompt, params, role)

data/lib/llm/providers/bedrock.rb

@@ -205,9 +205,8 @@ module LLM
       params = {role: :user, model: default_model, max_tokens: 2048}.merge!(params)
       tools = resolve_tools(params.delete(:tools))
       params = [params, adapt_schema(params), adapt_tools(tools)].inject({}, &:merge!).compact
-      role, stream = params.delete(:role), params.delete(:stream)
-      params[:stream] = true if streamable?(stream) || stream == true
-      [params, stream, tools, role]
+      role, stream = params.delete(:role), LLM::Stream.try(params.delete(:stream))
+      [params.merge!(stream: stream.enabled?), stream, tools, role]
     end
 
     def build_complete_request(prompt, params, role, stream: nil)
@@ -215,8 +214,8 @@ module LLM
       model_id = params.delete(:model) || default_model
       payload = build_converse_payload(messages, params)
       body = LLM.json.dump(payload)
-      path = stream ? "/model/#{model_id}/converse-stream" \
-                    : "/model/#{model_id}/converse"
+      path = stream&.enabled? ? "/model/#{model_id}/converse-stream" \
+                              : "/model/#{model_id}/converse"
       req = LLM::Transport::Request.post(path, headers)
       transport.set_body_stream(req, StringIO.new(body))
       [req, messages, body]

data/lib/llm/providers/google.rb

@@ -194,11 +194,15 @@ module LLM
     end
 
     def normalize_complete_params(params)
+      except = %i[role model messages stream]
       params = {role: :user, model: default_model}.merge!(params)
       tools  = resolve_tools(params.delete(:tools))
-      params = [params, adapt_generation_config(params), adapt_tools(tools)].inject({}, &:merge!).compact
-      role, model, stream = [:role, :model, :stream].map { params.delete(_1) }
-      [params, stream, tools, role, model]
+      config = adapt_generation_config(params.except(*except))
+      params = [params.except(:schema), config, adapt_tools(tools)].inject({}, &:merge!).compact
+      role, model, stream = params.delete(:role),
+                            params.delete(:model),
+                            LLM::Stream.try(params.delete(:stream))
+      [params.merge!(stream: stream.enabled?), stream, tools, role, model]
     end
 
     def build_complete_request(prompt, params, role, model, stream)

data/lib/llm/providers/mistral/request_adapter/completion.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module LLM::Mistral::RequestAdapter
+  ##
+  # @private
+  class Completion
+    ##
+    # @param [LLM::Message, Hash] message
+    #  The message to format
+    def initialize(message)
+      @message = message
+    end
+
+    ##
+    # Adapts the message for the Mistral chat completions API
+    # @return [Hash]
+    def adapt
+      catch(:abort) do
+        if Hash === message
+          {role: message[:role], content: adapt_content(message[:content])}
+        elsif message.tool_call?
+          {role: message.role, content: nil, tool_calls: message.extra[:original_tool_calls]}
+        else
+          adapt_message
+        end
+      end
+    end
+
+    private
+
+    def adapt_message
+      case content
+      when Array
+        adapt_array
+      else
+        {role: message.role, content: adapt_content(content)}
+      end
+    end
+
+    def adapt_array
+      if content.empty?
+        nil
+      elsif returns.any?
+        returns.map do
+          {
+            role: "tool",
+            name: _1.name,
+            tool_call_id: _1.id,
+            content: LLM.json.dump(_1.value)
+          }
+        end
+      else
+        {role: message.role, content: content.flat_map { adapt_content(_1) }}
+      end
+    end
+
+    def adapt_content(content)
+      case content
+      when LLM::Object
+        adapt_object(content)
+      when String
+        [{type: :text, text: content.to_s}]
+      when LLM::Response
+        adapt_remote_file(content)
+      when LLM::Message
+        adapt_content(content.content)
+      when LLM::Function::Return
+        throw(:abort, {
+          role: "tool",
+          name: content.name,
+          tool_call_id: content.id,
+          content: LLM.json.dump(content.value)
+        })
+      else
+        prompt_error!(content)
+      end
+    end
+
+    def adapt_object(object)
+      case object.kind
+      when :image_url
+        [{type: :image_url, image_url: {url: object.value}}]
+      when :local_file
+        adapt_local_file(object.value)
+      when :remote_file
+        adapt_remote_file(object.value)
+      else
+        prompt_error!(object)
+      end
+    end
+
+    def adapt_local_file(file)
+      if file.image?
+        [{type: :image_url, image_url: {url: file.to_data_uri}}]
+      else
+        [{type: :file, file: {filename: file.basename, file_data: file.to_data_uri}}]
+      end
+    end
+
+    def adapt_remote_file(file)
+      if file.file?
+        [{type: :file, file: {file_id: file.id}}]
+      else
+        prompt_error!(file)
+      end
+    end
+
+    def prompt_error!(content)
+      if LLM::Object === content
+        raise LLM::PromptError, "The given LLM::Object with kind '#{content.kind}' is not " \
+                                "supported by the Mistral chat completions API."
+      else
+        raise LLM::PromptError, "The given object (an instance of #{content.class}) " \
+                                "is not supported by the Mistral chat completions API."
+      end
+    end
+
+    def message = @message
+    def content = message.content
+    def returns = content.grep(LLM::Function::Return)
+  end
+end

data/lib/llm/providers/mistral/request_adapter.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class LLM::Mistral
+  ##
+  # @private
+  module RequestAdapter
+    require_relative "request_adapter/completion"
+    include LLM::OpenAI::RequestAdapter
+
+    ##
+    # @param [Array<LLM::Message>] messages
+    #  The messages to adapt
+    # @return [Array<Hash>]
+    def adapt(messages, mode: nil)
+      messages.filter_map do |message|
+        Completion.new(message).adapt
+      end
+    end
+  end
+end

data/lib/llm/providers/mistral.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+require_relative "openai" unless defined?(LLM::OpenAI)
+
+module LLM
+  ##
+  # The Mistral class implements a provider for
+  # [Mistral](https://mistral.ai) through its
+  # OpenAI-compatible API.
+  #
+  # @example
+  #   #!/usr/bin/env ruby
+  #   require "llm"
+  #
+  #   llm = LLM.mistral(key: ENV["KEY"])
+  #   ctx = LLM::Context.new(llm)
+  #   ctx.talk "Hello"
+  #   ctx.messages.select(&:assistant?).each { print "[#{_1.role}]", _1.content, "\n" }
+  class Mistral < OpenAI
+    require_relative "mistral/request_adapter"
+    include Mistral::RequestAdapter
+
+    HOST = "api.mistral.ai"
+    BASE_PATH = "/v1/"
+
+    ##
+    # @param key (see LLM::Provider#initialize)
+    # @param host (see LLM::Provider#initialize)
+    # @param base_path (see LLM::Provider#initialize)
+    # @return [LLM::Mistral]
+    def initialize(host: HOST, base_path: BASE_PATH, **)
+      super
+    end
+
+    ##
+    # @return [Symbol]
+    #  Returns the provider's name
+    def name
+      :mistral
+    end
+
+    ##
+    # @return [NotImplementedError]
+    def images
+      raise NotImplementedError
+    end
+
+    ##
+    # Provides an embedding.
+    # @param input (see LLM::Provider#embed)
+    # @param model (see LLM::Provider#embed)
+    # @param params (see LLM::Provider#embed)
+    # @raise (see LLM::Provider#request)
+    # @return (see LLM::Provider#embed)
+    def embed(input, model: "mistral-embed", **params)
+      super
+    end
+
+    ##
+    # Runs OCR on a remote image or document URL.
+    # @see https://docs.mistral.ai/api/endpoint/ocr#operation-ocr_v1_ocr_post Mistral OCR docs
+    # @param [String, nil] image_url
+    #  A remote HTTP(S) URL to the image
+    # @param [String, nil] document_url
+    #  A remote HTTP(S) URL to the document
+    # @param [String] model
+    #  The OCR model to use
+    # @param [Hash] params
+    #  Additional OCR parameters
+    # @raise (see LLM::Provider#request)
+    # @return [LLM::Response]
+    def ocr(image_url: nil, document_url: nil, model: "mistral-ocr-latest", **params)
+      if [image_url, document_url].all?(&:nil?)
+        raise ArgumentError, "must provide one of: image_url, document_url"
+      elsif [image_url, document_url].compact.size > 1
+        raise ArgumentError, "must provide one of: image_url, document_url"
+      end
+      document = parse_document(image_url, document_url)
+      req = LLM::Transport::Request.post("/v1/ocr", headers)
+      req.body = LLM.json.dump({model:, document:}.merge!(params))
+      res, = execute(request: req, operation: "ocr", model:)
+      LLM::Response.new(res)
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def responses
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [LLM::Mistral::Audio]
+    def audio
+      raise NotImplementedError
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def files
+      raise NotImplementedError
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def moderations
+      raise NotImplementedError
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def vector_stores
+      raise NotImplementedError
+    end
+
+    ##
+    # Returns the default model for chat completions
+    # @return [String]
+    def default_model
+      "mistral-large-latest"
+    end
+
+    private
+
+    ##
+    # @api private
+    def headers
+      lock do
+        (@headers || {}).merge(
+          "Authorization" => "Bearer #{@key}",
+          "Content-Type" => "application/json"
+        )
+      end
+    end
+
+    ##
+    # @api private
+    def parse_document(image_url, document_url)
+      if image_url
+        {type: "image_url", image_url:}
+      elsif document_url
+        {type: "document_url", document_url:}
+      end
+    end
+  end
+end

data/lib/llm/providers/ollama/response_adapter/completion.rb

@@ -9,6 +9,16 @@ module LLM::Ollama::ResponseAdapter
     end
     alias_method :choices, :messages
 
+    ##
+    # Returns the response body, parsing NDJSON when the
+    # transport returned a raw string (non-streaming path).
+    # @return [LLM::Object]
+    def body
+      raw = super
+      return raw unless String === raw
+      parse_ndjson(raw)
+    end
+
     ##
     # (see LLM::Contract::Completion#input_tokens)
     def input_tokens
@@ -110,6 +120,35 @@ module LLM::Ollama::ResponseAdapter
       end
     end
 
+    ##
+    # Parses a raw NDJSON string into an LLM::Object.
+    # Ollama's streaming API returns application/x-ndjson,
+    # but the transport only parses application/json. When
+    # the response body arrives as a raw string, we merge
+    # the NDJSON lines here.
+    # @param [String] raw
+    # @return [LLM::Object]
+    def parse_ndjson(raw)
+      lines = raw.split("\n").reject(&:empty?)
+      merged = lines.each_with_object({}) do |line, hash|
+        parsed = LLM.json.load(line)
+        next unless parsed.is_a?(Hash)
+        parsed.each do |key, value|
+          if key == "message" && value.is_a?(Hash)
+            if hash.key?("message")
+              hash["message"]["content"] << value["content"].to_s
+            else
+              hash["message"] = {"role" => value["role"], "content" => value["content"].to_s}
+              hash["message"]["tool_calls"] = value["tool_calls"] if value.key?("tool_calls")
+            end
+          else
+            hash[key] = value
+          end
+        end
+      end
+      LLM::Object.from(merged)
+    end
+
     include LLM::Contract::Completion
   end
 end

data/lib/llm/providers/ollama.rb

@@ -121,9 +121,8 @@ module LLM
       params = {role: :user, model: default_model, stream: true}.merge!(params)
       tools  = resolve_tools(params.delete(:tools))
       params = [params, {format: params[:schema]}, adapt_tools(tools)].inject({}, &:merge!).compact
-      role, stream = params.delete(:role), params.delete(:stream)
-      params[:stream] = true if streamable?(stream) || stream == true
-      [params, stream, tools, role]
+      role, stream = params.delete(:role), LLM::Stream.try(params.delete(:stream))
+      [params.merge!(stream: stream.enabled?), stream, tools, role]
     end
 
     def build_complete_request(prompt, params, role)

data/lib/llm/providers/openai/responses.rb

@@ -36,10 +36,13 @@ class LLM::OpenAI
     # @return [LLM::Response]
     def create(prompt, params = {})
       params = {role: :user, model: @provider.default_model}.merge!(params)
+      role, stream = params.delete(:role), LLM::Stream.try(params.delete(:stream))
       tools  = resolve_tools(params.delete(:tools))
-      params = [params, adapt_schema(params), adapt_tools(tools)].inject({}, &:merge!).compact
-      role, stream = params.delete(:role), params.delete(:stream)
-      params[:stream] = true if @provider.streamable?(stream) || stream == true
+      params = [
+        params.merge!(stream: stream.enabled?),
+        adapt_schema(params),
+        adapt_tools(tools)
+      ].inject({}, &:merge!).compact
       req = LLM::Transport::Request.post(path("/responses"), headers)
       messages = build_complete_messages(prompt, params, role)
       @provider.tracer.set_request_metadata(user_input: extract_user_input(messages, fallback: prompt))

data/lib/llm/providers/openai.rb

@@ -211,12 +211,11 @@ module LLM
       params = {role: :user, model: default_model}.merge!(params)
       tools = resolve_tools(params.delete(:tools))
       params = [params, adapt_schema(params), adapt_tools(tools)].inject({}, &:merge!).compact
-      role, stream = params.delete(:role), params.delete(:stream)
-      params[:stream] = true if streamable?(stream) || stream == true
+      role, stream = params.delete(:role), LLM::Stream.try(params.delete(:stream))
       if params[:stream]
         params[:stream_options] = {include_usage: true}.merge!(params[:stream_options] || {})
       end
-      [params, stream, tools, role]
+      [params.merge!(stream: stream.enabled?), stream, tools, role]
     end
 
     def build_complete_request(prompt, params, role)