ruby_llm 2.1.0 → 2.2.0

data/lib/ruby_llm/protocol/cache_lifetime.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class Protocol
+    # Prompt cache APIs spell lifetimes as short strings such as "5m" and
+    # "1h". A string column stores a Duration as its digits, so digits in a
+    # String count as seconds.
+    module CacheLifetime # :nodoc:
+      UNITS = { 'h' => 3600, 'm' => 60 }.freeze
+
+      module_function
+
+      def render(ttl)
+        seconds = ttl.is_a?(String) ? Float(ttl, exception: false)&.to_i : ttl.to_i
+        return ttl unless seconds
+
+        unit, size = UNITS.find { |_unit, length| (seconds % length).zero? }
+        unit ? "#{seconds / size}#{unit}" : "#{seconds}s"
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocol/native_content.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class Protocol
+    # Decides which provider-native parts of an assistant message, such as
+    # thinking signatures, raw content blocks, and continuations, replay to
+    # the model a request goes to.
+    module NativeContent # :nodoc:
+      private
+
+      # A thinking signature or a provider-shaped content block belongs to the
+      # model that produced it, and one provider can serve models that do not
+      # read each other's, so a message another model produced replays
+      # without them. A message with no known producer replays as it is.
+      def foreign_native_content?(message)
+        return false unless message.role == :assistant && carries_native_content?(message)
+
+        producer = producing_entry(message)
+        !producer.nil? && (producer.provider != @provider.slug || !current_model?(producer.model))
+      end
+
+      # Usage records the id a chat resolved its model to, and a message built
+      # by hand may name the model by an alias of it.
+      def current_model?(model_id)
+        return true if @model.nil? || model_id == @model.id
+
+        RubyLLM.models.find(model_id, provider: @provider.slug, config: @config).id == @model.id
+      rescue ModelNotFoundError
+        false
+      end
+
+      # Another model of this provider gets the raw content any of its models
+      # reads, such as a tool search, and none of the producing model's own.
+      def without_foreign_native_content(message)
+        portable = portable_raw_content(message) if producer_slug(message) == @provider.slug
+        message.without_native_content(raw_content: portable)
+      end
+
+      def portable_raw_content(_message)
+        nil
+      end
+
+      def carries_native_content?(message)
+        return true if message.thinking || message.raw_reasoning || message.raw_content || message.phase
+        return true if message.continuation
+
+        message.tool_call? && message.tool_calls.any?(&:thought_signature)
+      end
+
+      # Only a usage entry names the producer: a model id alone can belong
+      # to several providers.
+      def producer_slug(message)
+        producing_entry(message)&.provider
+      end
+
+      def producing_entry(message)
+        message.ruby_llm_usage_entries.reverse.find(&:succeeded?)
+      end
+
+      # A thinking signature can be another provider's opaque blob, and a
+      # message restored without its usage could come from anyone, so a
+      # protocol that verifies signatures gets back only its own.
+      def own_signature(message)
+        signature = message.thinking&.signature
+        signature if signature && producer_slug(message) == @provider.slug
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocol/replayed_arguments.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class Protocol
+    # Replayed provider content keeps the argument text of calls that were
+    # not a JSON object, and some providers reject such a request. These
+    # helpers swap that text for empty arguments.
+    module ReplayedArguments # :nodoc:
+      module_function
+
+      def malformed_call_ids(message)
+        message.tool_calls.filter_map { |call| call.provider_call_id if call.malformed_arguments? }
+      end
+
+      def blank(entries, message, id:, arguments:, empty:)
+        ids = malformed_call_ids(message)
+        return entries if ids.empty? || !entries.is_a?(Array)
+
+        entries.map do |entry|
+          next entry unless entry.is_a?(Hash) && entry.key?(arguments)
+          next entry unless Array(id).any? { |key| ids.include?(entry[key]) }
+
+          entry.merge(arguments => empty)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocol/stream_accumulator.rb

@@ -10,6 +10,7 @@ module RubyLLM
 
       def initialize
         @content = +''
+        @attachments = []
         @citations = []
         @citation_keys = {}
         @thinking_text = nil
@@ -20,8 +21,9 @@ module RubyLLM
         @raw_content = nil
         @raw_reasoning = nil
         @finish_reason = nil
-        @latest_tool_call_id = nil
-        @tool_call_ids_by_index = {}
+        @content_filters = {}
+        @latest_provider_call_id = nil
+        @provider_call_ids_by_index = {}
       end
 
       def add(chunk)
@@ -29,12 +31,13 @@ module RubyLLM
         @model = chunk.model if @model.to_s.empty?
 
         handle_chunk_content(chunk)
+        @attachments.concat(chunk.attachments)
         accumulate_citations(chunk.citations)
         append_thinking_from_chunk(chunk)
         accumulate_server_tool_calls(chunk.server_tool_calls)
-        @raw_content = chunk.raw_content if chunk.raw_content
-        @raw_reasoning = chunk.raw_reasoning if chunk.raw_reasoning
+        keep_native_fields(chunk)
         @finish_reason = chunk.finish_reason if chunk.finish_reason
+        accumulate_content_filters(chunk.content_filters)
         count_tokens chunk
         RubyLLM.logger.debug { inspect } if RubyLLM.config.log_stream_debug
       end
@@ -43,6 +46,7 @@ module RubyLLM
         Message.new(
           role: :assistant,
           content: content.empty? ? nil : content,
+          attachments: @attachments,
           citations: resolved_citations,
           thinking: Thinking.build(
             text: @thinking_text,
@@ -52,7 +56,10 @@ module RubyLLM
           server_tool_calls: @server_tool_calls,
           raw_content: @raw_content,
           raw_reasoning: @raw_reasoning,
+          phase: @phase,
+          continuation: @continuation,
           finish_reason: @finish_reason,
+          content_filters: @content_filters.values,
           model: model,
           tool_calls: tool_calls_from_stream(response),
           raw: response
@@ -61,6 +68,22 @@ module RubyLLM
 
       private
 
+      def keep_native_fields(chunk)
+        @raw_content = chunk.raw_content if chunk.raw_content
+        @raw_reasoning = chunk.raw_reasoning if chunk.raw_reasoning
+        @phase = chunk.phase if chunk.phase
+        @continuation = chunk.continuation if chunk.continuation
+      end
+
+      # A stream can judge the same category again as it goes; a verdict
+      # that filtered stays, and otherwise the latest one does.
+      def accumulate_content_filters(filters)
+        filters.each do |filter|
+          key = [filter.source, filter.category]
+          @content_filters[key] = filter unless @content_filters[key]&.filtered?
+        end
+      end
+
       # Providers like Perplexity repeat the full citation list on every chunk.
       def accumulate_citations(new_citations)
         new_citations.each do |citation|
@@ -98,53 +121,73 @@ module RubyLLM
 
       def tool_calls_from_stream(response)
         tool_calls.transform_values do |tc|
-          arguments = if tc.arguments.is_a?(String) && !tc.arguments.empty?
-                        parse_tool_call_arguments(tc.arguments, response)
-                      elsif tc.arguments.is_a?(String)
-                        {}
-                      else
-                        tc.arguments
-                      end
-
           ToolCall.new(
             id: tc.id,
+            provider_call_id: tc.provider_call_id,
             name: tc.name,
-            arguments: arguments,
+            namespace: tc.namespace,
+            **streamed_arguments(tc, response),
             thought_signature: tc.thought_signature,
             remote: tc.remote?
           )
         end
       end
 
-      def parse_tool_call_arguments(arguments, response)
-        JSON.parse(arguments)
-      rescue JSON::ParserError => e
-        raise ToolCallParseError.new(response: response, finish_reason: @finish_reason), cause: e
+      def streamed_arguments(tool_call, response)
+        arguments = tool_call.arguments
+        return { malformed_arguments: tool_call.malformed_arguments } if tool_call.malformed_arguments?
+        return { arguments: } unless arguments.is_a?(String)
+        raise ToolCallTruncatedError.new(response:) if arguments.empty? && cut_off?(tool_call)
+
+        ToolCall.parse_arguments(arguments, response:, finish_reason: @finish_reason)
+      end
+
+      def cut_off?(tool_call)
+        @finish_reason == :max_tokens && tool_calls.values.last.equal?(tool_call)
       end
 
       def accumulate_tool_calls(new_tool_calls)
         RubyLLM.logger.debug { "Accumulating tool calls: #{new_tool_calls}" } if RubyLLM.config.log_stream_debug
-        new_tool_calls.each do |stream_key, tool_call|
-          if tool_call.id
-            start_tool_call(stream_key, tool_call)
-          else
+        new_tool_calls.to_h.each do |stream_key, tool_call|
+          if continues_tool_call?(stream_key, tool_call)
             append_tool_call_fragment(stream_key, tool_call)
+          else
+            if tool_call.provider_call_id.to_s.empty?
+              tool_call = ToolCall.new(**tool_call.to_h, provider_call_id: SecureRandom.uuid)
+              new_tool_calls[stream_key] = tool_call
+            end
+            start_tool_call(stream_key, tool_call)
           end
         end
       end
 
+      def continues_tool_call?(stream_key, tool_call)
+        case tool_call.provider_call_id
+        when nil then tool_call.name.to_s.empty? || !find_tool_call(stream_key).nil?
+        when '' then tool_call.name.to_s.empty?
+        else
+          !tool_call.arguments.is_a?(Hash) && @provider_call_ids_by_index[stream_key] == tool_call.provider_call_id
+        end
+      end
+
       def start_tool_call(stream_key, tool_call)
-        tool_call_id = tool_call.id.empty? ? SecureRandom.uuid : tool_call.id
+        provider_call_id = tool_call.provider_call_id
+        if @tool_calls.key?(provider_call_id) && @provider_call_ids_by_index[stream_key] != provider_call_id
+          raise Error, "Duplicate tool call ID in one response: #{provider_call_id.inspect}"
+        end
 
-        @tool_calls[tool_call_id] = ToolCall.new(
-          id: tool_call_id,
+        @tool_calls[provider_call_id] = ToolCall.new(
+          id: tool_call.id,
+          provider_call_id:,
           name: tool_call.name,
+          namespace: tool_call.namespace,
           arguments: initial_tool_call_arguments(tool_call),
+          malformed_arguments: tool_call.malformed_arguments,
           thought_signature: tool_call.thought_signature,
           remote: tool_call.remote?
         )
-        @tool_call_ids_by_index[stream_key] = tool_call_id unless stream_key.nil?
-        @latest_tool_call_id = tool_call_id
+        @provider_call_ids_by_index[stream_key] = provider_call_id unless stream_key.nil?
+        @latest_provider_call_id = provider_call_id
       end
 
       def initial_tool_call_arguments(tool_call)
@@ -158,18 +201,16 @@ module RubyLLM
         existing = find_tool_call(stream_key)
         return unless existing
 
-        fragment = tool_call.arguments
-        fragment = '' if fragment.nil?
-        existing.arguments << fragment
+        existing.arguments << (tool_call.arguments || '') unless existing.arguments.is_a?(Hash)
         return unless tool_call.thought_signature && existing.thought_signature.nil?
 
         existing.thought_signature = tool_call.thought_signature
       end
 
       def find_tool_call(stream_key)
-        return @tool_calls[@latest_tool_call_id] if stream_key.nil?
+        return @tool_calls[@latest_provider_call_id] if stream_key.nil?
 
-        @tool_calls[@tool_call_ids_by_index[stream_key]] || @tool_calls[stream_key]
+        @tool_calls[@provider_call_ids_by_index[stream_key]] || @tool_calls[stream_key]
       end
 
       def count_tokens(chunk)
@@ -179,8 +220,7 @@ module RubyLLM
       def handle_chunk_content(chunk)
         accumulate_tool_calls(chunk.tool_calls) if chunk.tool_call?
 
-        content_text = chunk.content || ''
-        @content << (content_text.is_a?(String) ? content_text : content_text.to_s)
+        @content << chunk.content.to_s
       end
 
       def append_thinking_from_chunk(chunk)

data/lib/ruby_llm/protocol/streaming.rb

@@ -8,16 +8,48 @@ module RubyLLM
     module Streaming # :nodoc: all
       MAX_JSON_BODY_BYTES = 1024 * 1024
 
-      StreamState = Struct.new(:parser, :buffer, :json_body) do
+      StreamState = Struct.new(:parser, :buffer, :json_body, :stray_json, :stray_error) do
         def initialize
           super
           reset
         end
 
         def reset
-          self.parser = Transport::EventStreamParser.new
+          self.parser = Transport::EventStreamParser.new(on_unknown_line: method(:keep_stray_line))
           self.buffer = +''
           self.json_body = nil
+          self.stray_json = +''
+          self.stray_error = nil
+        end
+
+        # A provider can answer in the middle of a stream with a bare JSON
+        # error, which is not an event. Only a line that starts with "{" can
+        # open one, so other lines the stream doesn't understand can't hide it.
+        # Each object is read as its last line arrives, because more lines can
+        # follow it in the same read.
+        def keep_stray_line(line)
+          return if stray_error
+
+          self.stray_json = +'' if line.start_with?('{')
+          return if stray_json.empty? && !line.start_with?('{')
+          if stray_json.bytesize + line.bytesize > MAX_JSON_BODY_BYTES
+            raise Error, "Streaming JSON response exceeds #{MAX_JSON_BODY_BYTES} bytes"
+          end
+
+          text = line.force_encoding(Encoding::UTF_8).scrub
+          stray_json << text << "\n"
+          read_stray_json if text.rstrip.end_with?('}')
+        end
+
+        private
+
+        # A complete object without an error is a line to ignore, and an
+        # incomplete one may finish on a later line.
+        def read_stray_json
+          self.stray_error = stray_json if JSON.parse(stray_json)['error']
+          self.stray_json = +''
+        rescue JSON::ParserError
+          nil
         end
       end
 
@@ -25,42 +57,47 @@ module RubyLLM
 
       def stream_response(payload, additional_headers = {}, &block)
         accumulator = StreamAccumulator.new
+        received = false
 
         response = stream_events(stream_url, payload, additional_headers) do |data|
+          received = true
           chunk = build_chunk(data)
           accumulator.add chunk
           block.call chunk
         end
+        raise Error.new('Provider returned no stream events', response:) unless received
+
+        if (remainder = stream_remainder)
+          accumulator.add remainder
+          block.call remainder
+        end
 
         message = accumulator.to_message(response)
         RubyLLM.logger.debug { "Stream completed: #{message.content}" }
         message
       end
 
+      # Returns a last Chunk for text a protocol held back while streaming,
+      # or +nil+.
+      def stream_remainder
+        nil
+      end
+
       # Posts +payload+ to +url+ as a server-sent event stream, yielding each
       # parsed event Hash. Returns the Faraday response.
-      def stream_events(url, payload, additional_headers = {}, &block)
+      def stream_events(url, payload, additional_headers = {}, &handler)
         progress = {}
         fallback_state = StreamState.new
-        on_data = build_on_data_handler(progress, fallback_state: fallback_state) do |data|
-          block.call(data) if data.is_a?(Hash)
+        on_data = build_on_data_handler(progress, fallback_state: fallback_state) do |event|
+          handler.call(event)
+        rescue StandardError => e
+          progress[:callback_error] = e
+          raise
         end
 
-        @connection.post url, payload, usage: @usage_tracker, stream: true do |req|
+        post_stream(url, payload, on_data:, progress:, on_reset: fallback_state.method(:reset),
+                                  on_failure: stream_failure(fallback_state)) do |req|
           req.headers = additional_headers.merge(req.headers) unless additional_headers.empty?
-          (req.options.context ||= {})[Transport::Connection::STREAM_PROGRESS_KEY] = progress
-          req.options.context[Transport::ErrorMiddleware::STREAM_RESET_KEY] = fallback_state.method(:reset)
-          if faraday_1?
-            req.options[:on_data] = on_data
-          else
-            req.options.on_data = on_data
-          end
-        end
-      end
-
-      def handle_stream(&block)
-        build_on_data_handler do |data|
-          block.call(build_chunk(data)) if data.is_a?(Hash)
         end
       end
 
@@ -70,6 +107,21 @@ module RubyLLM
         Faraday::VERSION.start_with?('1')
       end
 
+      def post_stream(url, payload, on_data:, progress:, on_reset:, on_failure: nil)
+        @connection.post url, payload, usage: @usage_tracker, stream: true do |request|
+          yield request if block_given?
+          context = (request.options.context ||= {})
+          context[Transport::Connection::STREAM_PROGRESS_KEY] = progress
+          context[Transport::ErrorMiddleware::STREAM_RESET_KEY] = on_reset
+          context[Transport::ErrorMiddleware::STREAM_FAILURE_KEY] = on_failure if on_failure
+          request.options.on_data = on_data
+        end
+      end
+
+      def stream_failure(fallback_state)
+        ->(env) { raise_stream_failure(env[:streaming_state] || fallback_state, env) }
+      end
+
       # Some adapters omit the env, so ErrorMiddleware also resets the
       # request's fallback state before each attempt.
       def build_on_data_handler(progress = {}, fallback_state: StreamState.new, &handler)
@@ -96,10 +148,22 @@ module RubyLLM
         if json_body?(chunk, state)
           handle_json_body(chunk, state, env)
         else
+          keep_for_failure(chunk, state)
           handle_sse(chunk, state.parser, env, progress, &)
+          raise_stray_error(state, env)
         end
       end
 
+      def raise_stray_error(state, env)
+        raise_stream_error(state.stray_error, JSON.parse(state.stray_error), env) if state.stray_error
+      end
+
+      # Faraday 1 knows the status only once the response ends, so a failure
+      # body that is not JSON has to be kept until then.
+      def keep_for_failure(chunk, state)
+        state.buffer << chunk if faraday_1? && state.buffer.bytesize + chunk.bytesize <= MAX_JSON_BODY_BYTES
+      end
+
       # Adapters cut reads anywhere, so a read inside an event can start with
       # "{". Only the first bytes of a body can open a bare JSON error.
       def json_body?(chunk, state)
@@ -107,18 +171,35 @@ module RubyLLM
         state.json_body
       end
 
+      # Faraday 1 knows the status only once the response ends, so the error
+      # waits in the buffer until the error middleware reports it.
       def handle_json_body(chunk, state, env)
         if state.buffer.bytesize + chunk.bytesize > MAX_JSON_BODY_BYTES
           raise Error, "Streaming JSON response exceeds #{MAX_JSON_BODY_BYTES} bytes"
         end
 
         state.buffer << chunk
+        raise_buffered_error(state, env) unless faraday_1?
+      end
+
+      def raise_buffered_error(state, env)
         parsed = JSON.parse(state.buffer)
         raise_stream_error(state.buffer, parsed, env) if parsed.is_a?(Hash) && parsed['error']
       rescue JSON::ParserError
         RubyLLM.logger.debug { 'Waiting for the remaining JSON response bytes' }
       end
 
+      # Runs once the response has ended, when a body that never parsed as
+      # JSON is the only account of the failure.
+      def raise_stream_failure(state, env)
+        state.parser.finish
+        raise_stray_error(state, env)
+        raise_buffered_error(state, env)
+        return unless failed_http_status(env) && !state.buffer.empty?
+
+        env[:streaming_error_response] = build_stream_error_response(state.buffer.dup, env, nil)
+      end
+
       def handle_failed_response(chunk, buffer, env)
         return if buffer.bytesize + chunk.bytesize > MAX_JSON_BODY_BYTES
 
@@ -142,6 +223,8 @@ module RubyLLM
             next if data == '[DONE]'
 
             parsed = handle_data(data, env)
+            next unless parsed.is_a?(Hash)
+
             progress[:started] = true
             yield parsed
           end
@@ -150,7 +233,7 @@ module RubyLLM
 
       def handle_data(data, env)
         parsed = JSON.parse(data)
-        return parsed unless parsed.is_a?(Hash) && parsed.key?('error')
+        return parsed unless parsed.is_a?(Hash) && parsed['error']
 
         raise_stream_error(data, parsed, env)
       rescue JSON::ParserError => e
@@ -188,7 +271,7 @@ module RubyLLM
         error_status = failed_http_status(env) || status || 500
 
         if faraday_1? || env.nil?
-          Struct.new(:body, :status).new(parsed_data, error_status)
+          Struct.new(:body, :status, :response_headers).new(parsed_data, error_status, env&.response_headers)
         else
           # Faraday writes body into the request until a status is set.
           env.merge(status: error_status, body: parsed_data)
@@ -224,7 +307,7 @@ module RubyLLM
             # Some adapters do not expose response status during on_data.
             status = env&.status
 
-            if status && status != 200
+            if status && !status.between?(200, 299)
               on_failed_response.call(chunk, env)
             else
               on_chunk.call(chunk, env)

data/lib/ruby_llm/protocol/tool_namespaces.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class Protocol
+    # How a protocol carries Tool.namespace. A namespace the API has no
+    # native form for is folded into the function name,
+    # <tt>namespace_name</tt>, and the calls that come back under that name
+    # are mapped back onto the namespace and the tool's own name.
+    module ToolNamespaces
+      # Raises for a forced tool choice that names a namespaced tool, for
+      # protocols whose tool choice cannot name one.
+      def self.validate_tool_choice(tools, choice) # :nodoc:
+        return unless tools[choice]&.namespace
+
+        raise Error, "The namespaced tool #{choice} cannot be forced; use choice: :required instead"
+      end
+
+      # Returns the error a call answers with when an earlier call of its
+      # namespace in the same response failed, or +nil+ when the API runs
+      # each call on its own. Protocols whose client toolsets run a
+      # response's calls in order and stop at the first failure override
+      # this.
+      def skipped_tool_call_error(_tool_call)
+        nil
+      end
+
+      def native_tool_namespace?(_namespace) # :nodoc:
+        false
+      end
+
+      # Returns whether +call+ came back under a namespace the API's own tool
+      # search put a deferred function in, rather than one +tools+ declare.
+      # Protocols whose tool search does that override this.
+      def searched_tool_namespace?(_call, _tools) # :nodoc:
+        false
+      end
+
+      def complete(messages, tools:, **options, &block) # :nodoc:
+        restore_tool_namespaces(super, tools)
+      end
+
+      def render(messages, tools:, **) # :nodoc:
+        super(messages.map { |message| flatten_message(message) }, tools: flatten_tools(tools), **)
+      end
+
+      def count_tokens(messages, tools:, **) # :nodoc:
+        super(messages.map { |message| flatten_message(message) }, tools: flatten_tools(tools), **)
+      end
+
+      private
+
+      def flatten_tools(tools)
+        tools.transform_values { |tool| flattened?(tool) ? Tool::Flattened.wrap(tool) : tool }
+      end
+
+      def restore_tool_namespaces(message, tools)
+        return message unless message.is_a?(Message) && message.tool_call?
+
+        message.with_tool_calls(message.tool_calls.transform_values { |call| restore_tool_namespace(call, tools) })
+      end
+
+      def restore_tool_namespace(call, tools)
+        if call.namespace.nil?
+          tool = tools[call.name.to_sym]
+          tool && flattened?(tool) ? restore_tool_call(call, tool) : call
+        elsif searched_tool_namespace?(call, tools)
+          rebuild_tool_call(call, name: call.name, namespace: nil)
+        else
+          call
+        end
+      end
+
+      def flattened?(tool_or_call)
+        !tool_or_call.namespace.nil? && !native_tool_namespace?(tool_or_call.namespace)
+      end
+
+      def flatten_message(message)
+        return message unless message.tool_calls.any? { |call| flattened?(call) }
+
+        message.with_tool_calls(message.tool_calls.transform_values { |call| flatten_tool_call(call) })
+      end
+
+      def flatten_tool_call(call)
+        return call unless flattened?(call)
+
+        rebuild_tool_call(call, name: call.qualified_name, namespace: nil)
+      end
+
+      def restore_tool_call(call, tool)
+        rebuild_tool_call(call, name: tool.name, namespace: tool.namespace)
+      end
+
+      def rebuild_tool_call(call, name:, namespace:)
+        ToolCall.new(id: call.id, provider_call_id: call.provider_call_id, name:, namespace:,
+                     arguments: call.arguments, malformed_arguments: call.malformed_arguments,
+                     thought_signature: call.thought_signature, remote: call.remote?)
+      end
+    end
+  end
+end