llm.rb 4.10.0 → 4.11.1

data/lib/llm/providers/google.rb

@@ -18,6 +18,7 @@ module LLM
   #   ctx.talk ["Tell me about this photo", ctx.local_file("/images/photo.png")]
   #   ctx.messages.select(&:assistant?).each { print "[#{_1.role}]", _1.content, "\n" }
   class Google < Provider
+    require_relative "google/utils"
     require_relative "google/error_handler"
     require_relative "google/request_adapter"
     require_relative "google/response_adapter"
@@ -28,6 +29,7 @@ module LLM
     require_relative "google/files"
 
     include RequestAdapter
+    extend Utils
 
     HOST = "generativelanguage.googleapis.com"
 

data/lib/llm/providers/ollama/response_adapter/completion.rb

@@ -51,6 +51,12 @@ module LLM::Ollama::ResponseAdapter
       super
     end
 
+    ##
+    # (see LLM::Contract::Completion#reasoning_content)
+    def reasoning_content
+      super
+    end
+
     ##
     # (see LLM::Contract::Completion#content!)
     def content!

data/lib/llm/providers/ollama/stream_parser.rb

@@ -11,9 +11,9 @@ class LLM::Ollama
 
     ##
     # @return [LLM::OpenAI::Chunk]
-    def initialize(io)
+    def initialize(stream)
       @body = {}
-      @io = io
+      @stream = stream
     end
 
     ##
@@ -23,6 +23,12 @@ class LLM::Ollama
       tap { merge!(chunk) }
     end
 
+    ##
+    # Frees internal parser state used during streaming.
+    # @return [void]
+    def free
+    end
+
     private
 
     def merge!(chunk)
@@ -30,10 +36,10 @@ class LLM::Ollama
         if key == "message"
           if @body[key]
             @body[key]["content"] << value["content"]
-            @io << value["content"] if @io.respond_to?(:<<)
+            @stream << value["content"] if @stream.respond_to?(:<<)
           else
             @body[key] = value
-            @io << value["content"] if @io.respond_to?(:<<)
+            @stream << value["content"] if @stream.respond_to?(:<<)
           end
         else
           @body[key] = value

data/lib/llm/providers/ollama.rb

@@ -122,7 +122,7 @@ module LLM
       tools  = resolve_tools(params.delete(:tools))
       params = [params, {format: params[:schema]}, adapt_tools(tools)].inject({}, &:merge!).compact
       role, stream = params.delete(:role), params.delete(:stream)
-      params[:stream] = true if stream.respond_to?(:<<) || stream == true
+      params[:stream] = true if streamable?(stream) || stream == true
       [params, stream, tools, role]
     end
 

data/lib/llm/providers/openai/response_adapter/completion.rb

@@ -10,6 +10,7 @@ module LLM::OpenAI::ResponseAdapter
         extra = {
           index:, response: self,
           logprobs: choice.logprobs,
+          reasoning_content: message.reasoning_content,
           tool_calls: adapt_tool_calls(message.tool_calls),
           original_tool_calls: message.tool_calls
         }
@@ -63,6 +64,12 @@ module LLM::OpenAI::ResponseAdapter
       super
     end
 
+    ##
+    # (see LLM::Contract::Completion#reasoning_content)
+    def reasoning_content
+      super
+    end
+
     ##
     # (see LLM::Contract::Completion#content!)
     def content!

data/lib/llm/providers/openai/response_adapter/responds.rb

@@ -2,29 +2,101 @@
 
 module LLM::OpenAI::ResponseAdapter
   module Responds
-    def model = body.model
-    def response_id = respond_to?(:response) ? response["id"] : id
-    def choices = [adapt_message]
-    def annotations = choices[0].annotations
+    ##
+    # (see LLM::Contract::Completion#messages)
+    def messages
+      [adapt_message]
+    end
+    alias_method :choices, :messages
+
+    ##
+    # @return [String]
+    def response_id
+      respond_to?(:response) ? response["id"] : id
+    end
+
+    ##
+    # @return [Array<Hash>]
+    def annotations = messages[0].annotations
+
+    ##
+    # (see LLM::Contract::Completion#input_tokens)
+    def input_tokens
+      body.usage&.input_tokens || 0
+    end
+    alias_method :prompt_tokens, :input_tokens
+
+    ##
+    # (see LLM::Contract::Completion#output_tokens)
+    def output_tokens
+      body.usage&.output_tokens || 0
+    end
+    alias_method :completion_tokens, :output_tokens
+
+    ##
+    # (see LLM::Contract::Completion#reasoning_tokens)
+    def reasoning_tokens
+      body
+        .usage
+        &.output_tokens_details
+        &.reasoning_tokens || 0
+    end
+
+    ##
+    # (see LLM::Contract::Completion#total_tokens)
+    def total_tokens
+      body.usage&.total_tokens || 0
+    end
 
-    def prompt_tokens = body.usage&.input_tokens
-    def completion_tokens = body.usage&.output_tokens
-    def total_tokens = body.usage&.total_tokens
+    ##
+    # (see LLM::Contract::Completion#usage)
+    def usage
+      super
+    end
+
+    ##
+    # (see LLM::Contract::Completion#model)
+    def model
+      body.model
+    end
 
     ##
     # Returns the aggregated text content from the response outputs.
     # @return [String]
     def output_text
-      choices.find(&:assistant?).content || ""
+      content
+    end
+
+    ##
+    # (see LLM::Contract::Completion#content)
+    def content
+      super || ""
+    end
+
+    ##
+    # (see LLM::Contract::Completion#content!)
+    def content!
+      super
+    end
+
+    ##
+    # (see LLM::Contract::Completion#reasoning_content)
+    def reasoning_content
+      super
     end
 
     private
 
     def adapt_message
-      message = LLM::Message.new("assistant", +"", {response: self, tool_calls: []})
-      output.each.with_index do |choice, index|
+      message = LLM::Message.new("assistant", +"", {response: self, tool_calls: [], reasoning_content: +""})
+      output.each do |choice|
         if choice.type == "function_call"
           message.extra[:tool_calls] << adapt_tool(choice)
+        elsif choice.type == "reasoning"
+          (choice.summary || []).each do |summary|
+            next unless summary["type"] == "summary_text"
+            message.extra["reasoning_content"] << summary["text"]
+          end
         elsif choice.content
           choice.content.each do |c|
             next unless c["type"] == "output_text"
@@ -48,5 +120,7 @@ module LLM::OpenAI::ResponseAdapter
     rescue *LLM.json.parser_error
       {}
     end
+
+    include LLM::Contract::Completion
   end
 end

data/lib/llm/providers/openai/responses/stream_parser.rb

@@ -10,11 +10,13 @@ class LLM::OpenAI
     attr_reader :body
 
     ##
-    # @param [#<<] io An IO-like object
+    # @param [#<<, LLM::Stream] stream
+    #  A stream sink that implements {#<<} or the {LLM::Stream} interface
     # @return [LLM::OpenAI::Responses::StreamParser]
-    def initialize(io)
+    def initialize(stream)
       @body = {"output" => []}
-      @io = io
+      @stream = stream
+      @emits = {tools: []}
     end
 
     ##
@@ -24,6 +26,13 @@ class LLM::OpenAI
       tap { handle_event(chunk) }
     end
 
+    ##
+    # Frees internal parser state used during streaming.
+    # @return [void]
+    def free
+      @emits.clear
+    end
+
     private
 
     def handle_event(chunk)
@@ -56,9 +65,21 @@ class LLM::OpenAI
           if content_part && content_part["type"] == "output_text"
             content_part["text"] ||= ""
             content_part["text"] << delta_text
-            @io << delta_text if @io.respond_to?(:<<)
+            emit_content(delta_text)
           end
         end
+      when "response.function_call_arguments.delta"
+        output_item = @body["output"][chunk["output_index"]]
+        if output_item && output_item["type"] == "function_call"
+          output_item["arguments"] ||= +""
+          output_item["arguments"] << chunk["delta"]
+        end
+      when "response.function_call_arguments.done"
+        output_item = @body["output"][chunk["output_index"]]
+        if output_item && output_item["type"] == "function_call"
+          output_item["arguments"] = chunk["arguments"]
+          emit_tool(chunk["output_index"], output_item)
+        end
       when "response.output_item.done"
         output_index = chunk["output_index"]
         item = chunk["item"]
@@ -72,5 +93,43 @@ class LLM::OpenAI
         @body["output"][output_index]["content"][content_index] = part
       end
     end
+
+    def emit_content(value)
+      if @stream.respond_to?(:on_content)
+        @stream.on_content(value)
+      elsif @stream.respond_to?(:<<)
+        @stream << value
+      end
+    end
+
+    def emit_tool(index, tool)
+      return unless @stream.respond_to?(:on_tool_call)
+      return unless complete_tool?(tool)
+      return if @emits[:tools].include?(index)
+      function, error = resolve_tool(tool)
+      @emits[:tools] << index
+      @stream.on_tool_call(function, error)
+    end
+
+    def complete_tool?(tool)
+      tool["call_id"] && tool["name"] && parse_arguments(tool["arguments"])
+    end
+
+    def resolve_tool(tool)
+      registered = LLM::Function.find_by_name(tool["name"])
+      fn = (registered || LLM::Function.new(tool["name"])).dup.tap do |fn|
+        fn.id = tool["call_id"]
+        fn.arguments = parse_arguments(tool["arguments"])
+      end
+      [fn, (registered ? nil : @stream.tool_not_found(fn))]
+    end
+
+    def parse_arguments(arguments)
+      return nil if arguments.to_s.empty?
+      parsed = LLM.json.load(arguments)
+      Hash === parsed ? parsed : nil
+    rescue *LLM.json.parser_error
+      nil
+    end
   end
 end

data/lib/llm/providers/openai/responses.rb

@@ -39,7 +39,7 @@ class LLM::OpenAI
       tools  = resolve_tools(params.delete(:tools))
       params = [params, adapt_schema(params), adapt_tools(tools)].inject({}, &:merge!).compact
       role, stream = params.delete(:role), params.delete(:stream)
-      params[:stream] = true if stream.respond_to?(:<<) || stream == true
+      params[:stream] = true if @provider.streamable?(stream) || stream == true
       req = Net::HTTP::Post.new("/v1/responses", headers)
       messages = build_complete_messages(prompt, params, role)
       @provider.tracer.set_request_metadata(user_input: extract_user_input(messages, fallback: prompt))

data/lib/llm/providers/openai/stream_parser.rb

@@ -11,9 +11,10 @@ class LLM::OpenAI
 
     ##
     # @return [LLM::OpenAI::Chunk]
-    def initialize(io)
+    def initialize(stream)
       @body = {}
-      @io = io
+      @stream = stream
+      @emits = {tools: []}
     end
 
     ##
@@ -23,6 +24,13 @@ class LLM::OpenAI
       tap { merge!(chunk) }
     end
 
+    ##
+    # Frees internal parser state used during streaming.
+    # @return [void]
+    def free
+      @emits.clear
+    end
+
     private
 
     def merge!(chunk)
@@ -47,7 +55,11 @@ class LLM::OpenAI
             if key == "content"
               target_message[key] ||= +""
               target_message[key] << value
-              @io << value if @io.respond_to?(:<<)
+              emit_content(value)
+            elsif key == "reasoning_content"
+              target_message[key] ||= +""
+              target_message[key] << value
+              emit_reasoning_content(value)
             elsif key == "tool_calls"
               merge_tools!(target_message, value)
             else
@@ -60,8 +72,13 @@ class LLM::OpenAI
           (choice["delta"] || {}).each do |key, value|
             next if value.nil?
             if key == "content"
-              @io << value if @io.respond_to?(:<<)
+              emit_content(value)
+              message_hash[key] = value
+            elsif key == "reasoning_content"
+              emit_reasoning_content(value)
               message_hash[key] = value
+            elsif key == "tool_calls"
+              merge_tools!(message_hash, value)
             else
               message_hash[key] = value
             end
@@ -85,7 +102,54 @@ class LLM::OpenAI
         else
           target["tool_calls"][tindex] = toola
         end
+        emit_tool(target["tool_calls"][tindex], tindex)
+      end
+    end
+
+    def emit_content(value)
+      if @stream.respond_to?(:on_content)
+        @stream.on_content(value)
+      elsif @stream.respond_to?(:<<)
+        @stream << value
+      end
+    end
+
+    def emit_reasoning_content(value)
+      if @stream.respond_to?(:on_reasoning_content)
+        @stream.on_reasoning_content(value)
       end
     end
+
+    def emit_tool(tool, tindex)
+      return unless @stream.respond_to?(:on_tool_call)
+      return unless complete_tool?(tool)
+      return if @emits[:tools].include?(tindex)
+      function, error = resolve_tool(tool)
+      @emits[:tools] << tindex
+      @stream.on_tool_call(function, error)
+    end
+
+    def complete_tool?(tool)
+      function = tool["function"]
+      function && tool["id"] && function["name"] && parse_arguments(function["arguments"])
+    end
+
+    def resolve_tool(tool)
+      function = tool["function"]
+      registered = LLM::Function.find_by_name(function["name"])
+      fn = (registered || LLM::Function.new(function["name"])).dup.tap do |fn|
+        fn.id = tool["id"]
+        fn.arguments = parse_arguments(function["arguments"])
+      end
+      [fn, (registered ? nil : @stream.tool_not_found(fn))]
+    end
+
+    def parse_arguments(arguments)
+      return nil if arguments.to_s.empty?
+      parsed = LLM.json.load(arguments)
+      Hash === parsed ? parsed : nil
+    rescue *LLM.json.parser_error
+      nil
+    end
   end
 end

data/lib/llm/providers/openai.rb

@@ -212,7 +212,7 @@ module LLM
       tools = resolve_tools(params.delete(:tools))
       params = [params, adapt_schema(params), adapt_tools(tools)].inject({}, &:merge!).compact
       role, stream = params.delete(:role), params.delete(:stream)
-      params[:stream] = true if stream.respond_to?(:<<) || stream == true
+      params[:stream] = true if streamable?(stream) || stream == true
       if params[:stream]
         params[:stream_options] = {include_usage: true}.merge!(params[:stream_options] || {})
       end

data/lib/llm/stream/queue.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+class LLM::Stream
+  ##
+  # A small queue for collecting streamed tool work. Values can be immediate
+  # {LLM::Function::Return} objects or concurrent handles returned by
+  # {LLM::Function#spawn}. Calling {#wait(strategy)} resolves queued work and
+  # returns an array of {LLM::Function::Return} values.
+  class Queue
+    ##
+    # @return [LLM::Stream::Queue]
+    def initialize
+      @items = []
+    end
+
+    ##
+    # Enqueue a function return or spawned task.
+    # @param [LLM::Function::Return, Thread, Async::Task, Fiber] item
+    # @return [LLM::Stream::Queue]
+    def <<(item)
+      @items << item
+      self
+    end
+
+    ##
+    # Returns true when the queue is empty.
+    # @return [Boolean]
+    def empty?
+      @items.empty?
+    end
+
+    ##
+    # Waits for queued work to finish and returns function results.
+    # @param [Symbol] strategy
+    #   Controls concurrency strategy:
+    #   - `:thread`: Use threads
+    #   - `:task`: Use async tasks (requires async gem)
+    #   - `:fiber`: Use raw fibers
+    # @return [Array<LLM::Function::Return>]
+    def wait(strategy)
+      returns, tasks = @items.shift(@items.length).partition { LLM::Function::Return === _1 }
+      returns.concat case strategy
+      when :thread then LLM::Function::ThreadGroup.new(tasks).wait
+      when :task then LLM::Function::TaskGroup.new(tasks).wait
+      when :fiber then LLM::Function::FiberGroup.new(tasks).wait
+      else raise ArgumentError, "Unknown strategy: #{strategy.inspect}. Expected :thread, :task, or :fiber"
+      end
+    end
+    alias_method :value, :wait
+  end
+end

data/lib/llm/stream.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module LLM
+  ##
+  # The {LLM::Stream LLM::Stream} class provides the callback interface for
+  # streamed model output in llm.rb.
+  #
+  # A stream object can be an instance of {LLM::Stream LLM::Stream}, a
+  # subclass that overrides the callbacks it needs, or any other object that
+  # implements some or all of the same interface. {#queue} provides a small
+  # helper for collecting asynchronous tool work started from a callback, and
+  # {#tool_not_found} returns an in-band tool error when a streamed tool
+  # cannot be resolved.
+  #
+  # @note The `on_*` callbacks run inline with the streaming parser. They
+  #   therefore block streaming progress and should generally return as
+  #   quickly as possible.
+  #
+  # The most common callback is {#on_content}, which also maps to {#<<} for
+  # compatibility with `StringIO`-style objects. Providers may also call
+  # {#on_reasoning_content} and {#on_tool_call} when that data is available.
+  class Stream
+    require_relative "stream/queue"
+
+    ##
+    # Returns a lazily-initialized queue for tool results or spawned work.
+    # @return [LLM::Stream::Queue]
+    def queue
+      @queue ||= Queue.new
+    end
+
+    ##
+    # Waits for queued tool work to finish and returns function results.
+    # @param [Symbol] strategy
+    #  The concurrency strategy to use
+    # @return [Array<LLM::Function::Return>]
+    def wait(strategy)
+      queue.wait(strategy)
+    end
+
+    # @group Public callbacks
+
+    ##
+    # Called when visible assistant output is streamed.
+    # @param [String] content
+    #  A chunk of assistant-visible text.
+    # @return [nil]
+    def on_content(content)
+      nil
+    end
+    alias_method :<<, :on_content
+
+    ##
+    # Called when reasoning output is streamed separately from visible content.
+    # @param [String] content
+    #  A chunk of reasoning text.
+    # @return [nil]
+    def on_reasoning_content(content)
+      nil
+    end
+
+    ##
+    # Called when a streamed tool call has been fully constructed.
+    # @note A stream implementation may start tool execution here, for
+    #   example by pushing `tool.spawn(:thread)`, `tool.spawn(:fiber)`, or
+    #   `tool.spawn(:task)` onto {#queue}. When a streamed tool cannot be
+    #   resolved, `error` is passed as an {LLM::Function::Return}. It can be
+    #   sent back to the model, allowing the tool-call path to recover and the
+    #   session to continue. Tool resolution depends on
+    #   {LLM::Function.registry}, which includes {LLM::Tool LLM::Tool}
+    #   subclasses, including MCP tools, but not functions defined with
+    #   {LLM.function}.
+    # @param [LLM::Function] tool
+    #  The parsed tool call.
+    # @param [LLM::Function::Return, nil] error
+    #  An in-band tool error for unresolved tool calls.
+    # @return [nil]
+    def on_tool_call(tool, error)
+      nil
+    end
+
+    # @endgroup
+
+    # @group Error handlers
+
+    ##
+    # Returns a function return describing a streamed tool that could not
+    # be resolved.
+    # @note This is mainly useful as a fallback from {#on_tool_call}. It
+    #   should be uncommon in normal use, since streamed tool callbacks only
+    #   run for tools already defined in the context.
+    # @param [LLM::Function] tool
+    # @return [LLM::Function::Return]
+    def tool_not_found(tool)
+      LLM::Function::Return.new(tool.id, tool.name, {
+        error: true, type: LLM::NoSuchToolError.name, message: "tool not found"
+      })
+    end
+
+    # @endgroup
+  end
+end