llm.rb 4.6.0 → 4.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c26db042940fc118505804bbec91d43dd39313b6625b4e3d7df51f43f01d58ae
-  data.tar.gz: 5780f3e3d0db3579772d4f7ba141052314d34b7fd0db06ba029e4b4fd6fd84dc
+  metadata.gz: 696d0be686c58d66ce0904d0ed0aff879906511060704c61adc945b458ed1f37
+  data.tar.gz: 7d03d786ff1fdbaa24e470f87d6e0c3c6b5acfc8e6b73c1cda88fdb9c1a4db58
 SHA512:
-  metadata.gz: 864ba517e81d8fb4d248d754c5a9fc4bdcbf162f9ac4fb7691a54c015006ed40035300fd19c2057f755678676534a568204b0e0bc29b8e058cf2334b8625465e
-  data.tar.gz: cd8b18c4e780a26c83201ea39f81077ad2535db966efcb5fe860f4af55808f96a8bd8a6bdd4c25f38a9012b3092eb2611bbfd31552969b00b15579af5e95354a
+  metadata.gz: aeb39e7a8b7a9826be90f3f4739ae0fae40df6bc6c7053e77110ccf1ebe9c5f44316b0edf0e2de3782d0fa93dce110ac8cc0328623056b6c71e55be3abf792c3
+  data.tar.gz: c665f150d6c12cbe27541ea743a0fbea4eb107eb5bc93d4e9af480460108430a3057852458f9c5bd1fa5d17ed67633462513ff098c02f0fef43de9dc5ab44224

data/README.md

@@ -4,7 +4,7 @@
 <p align="center">
   <a href="https://0x1eef.github.io/x/llm.rb?rebuild=1"><img src="https://img.shields.io/badge/docs-0x1eef.github.io-blue.svg" alt="RubyDoc"></a>
   <a href="https://opensource.org/license/0bsd"><img src="https://img.shields.io/badge/License-0BSD-orange.svg?" alt="License"></a>
-  <a href="https://github.com/llmrb/llm.rb/tags"><img src="https://img.shields.io/badge/version-4.6.0-green.svg?" alt="Version"></a>
+  <a href="https://github.com/llmrb/llm.rb/tags"><img src="https://img.shields.io/badge/version-4.8.0-green.svg?" alt="Version"></a>
 </p>
 
 ## About
@@ -66,7 +66,7 @@ end
 llm = LLM.openai(key: ENV["KEY"])
 ses = LLM::Session.new(llm, schema: Report)
 res = ses.talk("Structure this report: 'Database latency spiked at 10:42 UTC, causing 5% request timeouts for 12 minutes.'")
-pp res.messages.first(&:assistant?).content!
+pp res.content!
 
 ##
 # {
@@ -181,31 +181,41 @@ ses.talk(prompt)
 
 llm.rb is designed for threaded environments with throughput in mind.
 Locks are used selectively, and localized state is preferred wherever
-possible. Blanket locking across every class would help guarantee
-correctness but it would also add contention, reduce throughput,
+possible. Blanket locking across every class could help guarantee
+correctness but it could also add contention, reduce throughput,
 and increase complexity.
 
 That's why we decided to optimize for both correctness and throughput
 instead. An important part of that design is guaranteeing that
 [LLM::Provider](https://0x1eef.github.io/x/llm.rb/LLM/Provider.html)
-is safe to share across threads. [LLM::Session](https://0x1eef.github.io/x/llm.rb/LLM/Session.html) and
+is safe to share and use across threads. [LLM::Session](https://0x1eef.github.io/x/llm.rb/LLM/Session.html) and
 [LLM::Agent](https://0x1eef.github.io/x/llm.rb/LLM/Agent.html) are
-stateful objects that should be kept local to a single thread. So the
-recommended pattern is to keep one session or agent per thread,
-and share a provider across multiple threads:
+stateful objects that should be kept local to a single thread.
+
+[LLM::Tracer](https://0x1eef.github.io/x/llm.rb/LLM/Tracer.html) and its
+subclasses are also designed to be thread-local, which means that
+`llm.tracer = ...` only impacts the current thread and must be set
+again in each thread where a tracer is desired. This avoids contention
+on tracer state, keeps tracing isolated per thread, and allows different
+tracers to be used in different threads simultaneously.
+
+So the recommended pattern is to keep one session, tracer or agent per
+thread, and share a provider across multiple threads:
+
 
 ```ruby
 #!/usr/bin/env ruby
 require "llm"
 
-llm = LLM.openai(key: ENV["KEY"], persistent: true)
+llm = LLM.openai(key: ENV["KEY"]).persist!
 schema = llm.schema.object(answer: llm.schema.integer.required)
 
 vals = 10.times.map do |x|
   Thread.new do
+    llm.tracer = LLM::Tracer::Logger.new(llm, path: "thread#{x}.log")
     ses = LLM::Session.new(llm, schema:)
     res = ses.talk "#{x} + 5 = ?"
-    res.messages.find(&:assistant?).content!
+    res.content!
   end
 end.map(&:value)
 
@@ -335,7 +345,7 @@ and the gem should be installed separately:
 #!/usr/bin/env ruby
 require "llm"
 
-llm  = LLM.openai(key: ENV["KEY"], persistent: true)
+llm  = LLM.openai(key: ENV["KEY"]).persist!
 res1 = llm.responses.create "message 1"
 res2 = llm.responses.create "message 2", previous_response_id: res1.response_id
 res3 = llm.responses.create "message 3", previous_response_id: res2.response_id
@@ -349,6 +359,11 @@ can be used to trace LLM requests. It can be useful for debugging, monitoring,
 and observability. The primary use case in mind is integration with tools like
 [LangSmith](https://www.langsmith.com/).
 
+It is worth mentioning that tracers are local to a thread, and they
+should be configured per thread. That means that `llm.tracer = LLM::Tracer::Telemetry.new(llm)`
+only impacts the current thread, and it should be repeated in each thread where
+tracing is required.
+
 The telemetry implementation uses the [opentelemetry-sdk](https://github.com/open-telemetry/opentelemetry-ruby)
 and is based on the [gen-ai telemetry spec(s)](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/).
 This feature is optional, disabled by default, and the [opentelemetry-sdk](https://github.com/open-telemetry/opentelemetry-ruby)
@@ -406,7 +421,8 @@ The llm.rb library includes simple logging support through its
 tracer API, and Ruby's standard library ([ruby/logger](https://github.com/ruby/logger)).
 This feature is optional, disabled by default, and it can be useful for debugging and/or
 monitoring requests to LLM providers. The `path` or `io` options can be used to choose
-where logs are written to, and by default it is set to `$stdout`:
+where logs are written, and by default it is set to `$stdout`. Like other tracers,
+the logger tracer is local to a thread:
 
 ```ruby
 #!/usr/bin/env ruby
@@ -578,12 +594,13 @@ it has been uploaded. The file (a specialized instance of
 ```ruby
 #!/usr/bin/env ruby
 require "llm"
+require "pp"
 
 llm = LLM.openai(key: ENV["KEY"])
 ses = LLM::Session.new(llm)
 file = llm.files.create(file: "/tmp/llm-book.pdf")
 res = ses.talk ["Tell me about this file", file]
-res.messages.each { |m| puts "[#{m.role}] #{m.content}" }
+pp res.content
 ```
 
 ### Prompts
@@ -674,68 +691,53 @@ puts res.text # => "Good morning."
 Some but not all LLM providers implement image generation capabilities that
 can create new images from a prompt, or edit an existing image with a
 prompt. The following example uses the OpenAI provider to create an
-image of a dog on a rocket to the moon. The image is then moved to
+image of a dog on a rocket to the moon. The image is then written to
 `${HOME}/dogonrocket.png` as the final step:
 
 ```ruby
 #!/usr/bin/env ruby
 require "llm"
-require "open-uri"
-require "fileutils"
-
 llm = LLM.openai(key: ENV["KEY"])
 res = llm.images.create(prompt: "a dog on a rocket to the moon")
-res.urls.each do |url|
-  FileUtils.mv OpenURI.open_uri(url).path,
-               File.join(Dir.home, "dogonrocket.png")
-end
+IO.copy_stream res.images[0], File.join(Dir.home, "dogonrocket.png")
 ```
 
 #### Edit
 
 The following example is focused on editing a local image with the aid
 of a prompt. The image (`/tmp/llm-logo.png`) is returned to us with a hat.
-The image is then moved to `${HOME}/logo-with-hat.png` as
+The image is then written to `${HOME}/logo-with-hat.png` as
 the final step:
 
 ```ruby
 #!/usr/bin/env ruby
 require "llm"
-require "open-uri"
-require "fileutils"
-
 llm = LLM.openai(key: ENV["KEY"])
 res = llm.images.edit(
   image: "/tmp/llm-logo.png",
   prompt: "add a hat to the logo",
 )
-res.urls.each do |url|
-  FileUtils.mv OpenURI.open_uri(url).path,
-               File.join(Dir.home, "logo-with-hat.png")
-end
+IO.copy_stream res.images[0], File.join(Dir.home, "logo-with-hat.png")
 ```
 
 #### Variations
 
 The following example is focused on creating variations of a local image.
 The image (`/tmp/llm-logo.png`) is returned to us with five different variations.
-The images are then moved to `${HOME}/logo-variation0.png`, `${HOME}/logo-variation1.png`
+The images are then written to `${HOME}/logo-variation0.png`, `${HOME}/logo-variation1.png`
 and so on as the final step:
 
 ```ruby
 #!/usr/bin/env ruby
 require "llm"
-require "open-uri"
-require "fileutils"
-
 llm = LLM.openai(key: ENV["KEY"])
 res = llm.images.create_variation(
   image: "/tmp/llm-logo.png",
   n: 5
 )
-res.urls.each.with_index do |url, index|
-  FileUtils.mv OpenURI.open_uri(url).path,
-               File.join(Dir.home, "logo-variation#{index}.png")
+res.images.each.with_index do |image, index|
+  IO.copy_stream image,
+                 File.join(Dir.home, "logo-variation#{index}.png")
 end
 ```
 
@@ -776,6 +778,7 @@ release:
 ```ruby
 #!/usr/bin/env ruby
 require "llm"
+require "pp"
 
 ##
 # List all models
@@ -789,7 +792,7 @@ end
 model = llm.models.all.find { |m| m.id == "gpt-3.5-turbo" }
 ses = LLM::Session.new(llm, model: model.id)
 res = ses.talk "Hello #{model.id} :)"
-res.messages.each { |m| puts "[#{m.role}] #{m.content}" }
+pp res.content
 ```
 
 ## Install

data/lib/llm/contract/completion.rb

@@ -43,6 +43,20 @@ module LLM::Contract
       raise NotImplementedError, "#{self.class} does not implement '#{__method__}'"
     end
 
+    ##
+    # @return [String]
+    #  Returns the LLM response
+    def content
+      messages.find(&:assistant?).content
+    end
+
+    ##
+    # @return [Hash]
+    #  Returns the LLM response after parsing it as JSON
+    def content!
+      LLM.json.load(content)
+    end
+
     ##
     # @return [LLM::Usage]
     #  Returns usage information

data/lib/llm/eventstream/parser.rb

@@ -80,11 +80,6 @@ module LLM::EventStream
         @cursor = newline + 1
         yield(line)
       end
-      if @cursor < @buffer.length
-        line = @buffer[@cursor..]
-        @cursor = @buffer.length
-        yield(line)
-      end
       return if @cursor.zero?
       @buffer = @buffer[@cursor..] || +""
       @cursor = 0

data/lib/llm/model.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+##
+# The {LLM::Model LLM::Model} class provides a normalized view of
+# a provider model record returned by the models API.
+class LLM::Model
+  ##
+  # The provider-specific model payload.
+  # @return [LLM::Object]
+  attr_reader :raw
+
+  ##
+  # @param [LLM::Object, Hash] raw
+  def initialize(raw)
+    @raw = raw
+  end
+
+  ##
+  # Returns a normalized identifier suitable for API calls.
+  # @return [String, nil]
+  def id
+    normalize_id(raw.id || raw.model || raw.name)
+  end
+
+  ##
+  # Returns a display-friendly model name.
+  # @return [String, nil]
+  def name
+    raw.display_name || raw.displayName || id
+  end
+
+  ##
+  # Best-effort predicate for chat support.
+  # @return [Boolean]
+  def chat?
+    return true if anthropic?
+    return [*(raw.supportedGenerationMethods || [])].include?("generateContent") if gemini?
+    openai_compatible_chat?
+  end
+
+  ##
+  # Returns a Hash representation of the normalized model.
+  # @return [Hash]
+  def to_h
+    {id:, name:, chat?: chat?}.compact
+  end
+
+  ##
+  # @private
+  module Collection
+    include ::Enumerable
+
+    ##
+    # @yield [model]
+    # @yieldparam [LLM::Model] model
+    # @return [Enumerator, void]
+    def each(&)
+      return enum_for(:each) unless block_given?
+      models.each(&)
+    end
+
+    ##
+    # Returns an element, or a slice, or nil.
+    # @return [Object, Array<Object>, nil]
+    def [](*pos, **kw)
+      models[*pos, **kw]
+    end
+
+    ##
+    # @return [Boolean]
+    def empty?
+      models.empty?
+    end
+
+    ##
+    # @return [Integer]
+    def size
+      models.size
+    end
+
+    ##
+    # Returns normalized models.
+    # @return [Array<LLM::Model>]
+    def models
+      @models ||= raw_models.map { LLM::Model.new(_1) }
+    end
+  end
+
+  private
+
+  def normalize_id(value)
+    value&.sub(%r{\Amodels/}, "")
+  end
+
+  def anthropic?
+    raw.type == "model" && raw.key?(:display_name) && raw.key?(:created_at)
+  end
+
+  def gemini?
+    raw.key?(:supportedGenerationMethods)
+  end
+
+  def openai_compatible_chat?
+    value = [id, raw.name, raw.model].compact.join(" ").downcase
+    return false if value.include?("embedding")
+    return false if value.include?("moderation")
+    return false if value.include?("tts")
+    return false if value.include?("transcrib")
+    return false if value.include?("image")
+    return false if value.include?("whisper")
+    return false if value.include?("dall")
+    return false if value.include?("omni-moderation")
+    true
+  end
+end

data/lib/llm/provider.rb

@@ -37,7 +37,6 @@ class LLM::Provider
     @timeout = timeout
     @ssl = ssl
     @client = persistent ? persistent_client : nil
-    @tracer = LLM::Tracer::Null.new(self)
     @base_uri = URI("#{ssl ? "https" : "http"}://#{host}:#{port}/")
     @headers = {"User-Agent" => "llm.rb v#{LLM::VERSION}"}
     @monitor = Monitor.new
@@ -48,7 +47,7 @@ class LLM::Provider
   # @return [String]
   # @note The secret key is redacted in inspect for security reasons
   def inspect
-    "#<#{self.class.name}:0x#{object_id.to_s(16)} @key=[REDACTED] @client=#{@client.inspect} @tracer=#{@tracer.inspect}>"
+    "#<#{self.class.name}:0x#{object_id.to_s(16)} @key=[REDACTED] @client=#{@client.inspect} @tracer=#{tracer.inspect}>"
   end
 
   ##
@@ -265,27 +264,48 @@ class LLM::Provider
 
   ##
   # @return [LLM::Tracer]
-  #  Returns an LLM tracer
+  #  Returns a thread-local tracer
   def tracer
-    @tracer
+    weakmap[self] || LLM::Tracer::Null.new(self)
   end
 
   ##
-  # Set the tracer
+  # Set a thread-local tracer
   # @example
   #   llm = LLM.openai(key: ENV["KEY"])
-  #   llm.tracer = LLM::Tracer::Logger.new(llm, path: "/path/to/log.txt")
+  #   Thread.new do
+  #     llm.tracer = LLM::Tracer::Logger.new(llm, path: "/path/to/log/1.txt")
+  #   end
+  #   Thread.new do
+  #     llm.tracer = LLM::Tracer::Logger.new(llm, path: "/path/to/log/2.txt")
+  #   end
   #   # ...
   # @param [LLM::Tracer] tracer
   #  A tracer
   # @return [void]
   def tracer=(tracer)
-    lock do
-      @tracer = if tracer.nil?
-        LLM::Tracer::Null.new(self)
+    if tracer.nil?
+      if weakmap.respond_to?(:delete)
+        weakmap.delete(self)
       else
-        tracer
+        weakmap[self] = nil
       end
+    else
+      weakmap[self] = tracer
+    end
+  end
+
+  ##
+  # This method configures a provider to use a persistent connection pool
+  # via the optional dependency [Net::HTTP::Persistent](https://github.com/drbrain/net-http-persistent)
+  # @example
+  #   llm = LLM.openai(key: ENV["KEY"]).persist!
+  #   # do something with 'llm'
+  # @return [LLM::Provider]
+  def persist!
+    client = persistent_client
+    lock do
+      tap { @client = client }
     end
   end
 
@@ -340,9 +360,9 @@ class LLM::Provider
   # @raise [SystemCallError]
   #  When there is a network error at the operating system level
   # @return [Net::HTTPResponse]
-  def execute(request:, operation:, stream: nil, stream_parser: self.stream_parser, model: nil, &b)
-    tracer = @tracer
-    span = tracer.on_request_start(operation:, model:)
+  def execute(request:, operation:, stream: nil, stream_parser: self.stream_parser, model: nil, inputs: nil, &b)
+    tracer = self.tracer
+    span = tracer.on_request_start(operation:, model:, inputs:)
     http = client || transient_client
     args = (Net::HTTP === http) ? [request] : [URI.join(base_uri, request.path), request]
     res = if stream
@@ -351,11 +371,12 @@ class LLM::Provider
         parser = LLM::EventStream::Parser.new
         parser.register(handler)
         res.read_body(parser)
-        # If the handler body is empty, it means the
-        # response was most likely not streamed or
-        # parsing has failed. In that case, we fallback
-        # on the original response body.
-        res.body = LLM::Object.from(handler.body.empty? ? parser.body : handler.body)
+        # If the handler body is empty, the response was
+        # most likely not streamed or parsing failed.
+        # Preserve the raw body in that case so standard
+        # JSON/error handling can parse it later.
+        body = handler.body.empty? ? parser.body : handler.body
+        res.body = Hash === body || Array === body ? LLM::Object.from(body) : body
       ensure
         parser&.free
       end
@@ -423,14 +444,20 @@ class LLM::Provider
   end
 
   ##
-  # @return [Hash<Symbol, LLM::Tracer>]
-  def tracers
-    self.class.tracers
+  # @api private
+  def lock(&)
+    @monitor.synchronize(&)
   end
 
   ##
   # @api private
-  def lock(&)
-    @monitor.synchronize(&)
+  def thread
+    Thread.current
+  end
+
+  ##
+  # @api private
+  def weakmap
+    thread[:"llm.provider.weakmap"] ||= ObjectSpace::WeakMap.new
   end
 end

data/lib/llm/providers/anthropic/error_handler.rb

@@ -35,7 +35,7 @@ class LLM::Anthropic
       ex = error
       @tracer.on_request_error(ex:, span:)
     ensure
-      raise(ex)
+      raise(ex) if ex
     end
 
     private

data/lib/llm/providers/anthropic/models.rb

@@ -41,7 +41,7 @@ class LLM::Anthropic
       query = URI.encode_www_form(params)
       req = Net::HTTP::Get.new("/v1/models?#{query}", headers)
       res, span, tracer = execute(request: req, operation: "request")
-      res = ResponseAdapter.adapt(res, type: :enumerable)
+      res = ResponseAdapter.adapt(res, type: :models)
       tracer.on_request_finish(operation: "request", res:, span:)
       res
     end

data/lib/llm/providers/anthropic/request_adapter.rb

@@ -9,11 +9,20 @@ class LLM::Anthropic
     ##
     # @param [Array<LLM::Message>] messages
     #  The messages to adapt
-    # @return [Array<Hash>]
+    # @return [Hash]
     def adapt(messages, mode: nil)
-      messages.filter_map do
-        Completion.new(_1).adapt
+      payload = {messages: [], system: []}
+      messages.each do |message|
+        adapted = Completion.new(message).adapt
+        next if adapted.nil?
+        if system?(message)
+          payload[:system].concat Array(adapted[:content])
+        else
+          payload[:messages] << adapted
+        end
       end
+      payload.delete(:system) if payload[:system].empty?
+      payload
     end
 
     private
@@ -25,5 +34,13 @@ class LLM::Anthropic
       return {} unless tools&.any?
       {tools: tools.map { _1.respond_to?(:adapt) ? _1.adapt(self) : _1 }}
     end
+
+    def system?(message)
+      if message.respond_to?(:system?)
+        message.system?
+      else
+        Hash === message and message[:role].to_s == "system"
+      end
+    end
   end
 end

data/lib/llm/providers/anthropic/response_adapter/completion.rb

@@ -45,6 +45,18 @@ module LLM::Anthropic::ResponseAdapter
       body.model
     end
 
+    ##
+    # (see LLM::Contract::Completion#content)
+    def content
+      super
+    end
+
+    ##
+    # (see LLM::Contract::Completion#content!)
+    def content!
+      super
+    end
+
     private
 
     def adapt_choices

data/lib/llm/providers/anthropic/response_adapter/models.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module LLM::Anthropic::ResponseAdapter
+  module Models
+    include LLM::Model::Collection
+
+    private
+
+    def raw_models
+      data || []
+    end
+  end
+end

data/lib/llm/providers/anthropic/response_adapter.rb

@@ -7,6 +7,7 @@ class LLM::Anthropic
     require_relative "response_adapter/completion"
     require_relative "response_adapter/enumerable"
     require_relative "response_adapter/file"
+    require_relative "response_adapter/models"
     require_relative "response_adapter/web_search"
 
     module_function
@@ -27,6 +28,7 @@ class LLM::Anthropic
       when :completion then LLM::Anthropic::ResponseAdapter::Completion
       when :enumerable then LLM::Anthropic::ResponseAdapter::Enumerable
       when :file then LLM::Anthropic::ResponseAdapter::File
+      when :models then LLM::Anthropic::ResponseAdapter::Models
       when :web_search then LLM::Anthropic::ResponseAdapter::WebSearch
       else
         raise ArgumentError, "Unknown response adapter type: #{type.inspect}"

data/lib/llm/providers/anthropic.rb

@@ -140,7 +140,8 @@ module LLM
 
     def build_complete_request(prompt, params, role)
       messages = [*(params.delete(:messages) || []), Message.new(role, prompt)]
-      body = LLM.json.dump({messages: [adapt(messages)].flatten}.merge!(params))
+      payload = adapt(messages)
+      body = LLM.json.dump(payload.merge!(params))
       req = Net::HTTP::Post.new("/v1/messages", headers)
       set_body_stream(req, StringIO.new(body))
       req

data/lib/llm/providers/gemini/error_handler.rb

@@ -35,15 +35,15 @@ class LLM::Gemini
       ex = error
       @tracer.on_request_error(ex:, span:)
     ensure
-      raise(ex)
+      raise(ex) if ex
     end
 
     private
 
     ##
-    # @return [LLM::Object]
+    # @return [String, LLM::Object]
     def body
-      @body ||= LLM.json.load(res.body)
+      @body ||= parse_body!
     end
 
     ##
@@ -65,5 +65,20 @@ class LLM::Gemini
         LLM::Error.new("Unexpected response").tap { _1.response = res }
       end
     end
+
+    ##
+    # Tries to parse the response body as a LLM::Object
+    # @return [String, LLM::Object]
+    def parse_body!
+      if String === res.body
+        LLM::Object.from LLM.json.load(res.body)
+      elsif Hash === res.body
+        LLM::Object.from(res.body)
+      else
+        res.body
+      end
+    rescue
+      res.body
+    end
   end
 end

data/lib/llm/providers/gemini/response_adapter/completion.rb

@@ -45,6 +45,18 @@ module LLM::Gemini::ResponseAdapter
       body.modelVersion
     end
 
+    ##
+    # (see LLM::Contract::Completion#content)
+    def content
+      super
+    end
+
+    ##
+    # (see LLM::Contract::Completion#content!)
+    def content!
+      super
+    end
+
     private
 
     def adapt_choices