llm.rb 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5cd6331d31fab0e7582d9a0bef7a10b45e910fff70faace6bae774dd2757cff6
-  data.tar.gz: 42f0d68055c3f4aa732bb31b7dddd8d72b8e4fa90ba8727e871a43b736060e8f
+  metadata.gz: a7f9330df025e16999629b094202d78b504de9bedeafc45ec8d75bf729024567
+  data.tar.gz: e3a22daa5ac7add9b815346fe810ed368b9f3e3a3bb135cb4e5bbbb3875eacae
 SHA512:
-  metadata.gz: b3158f6d82d9f344deef6727ae2e74f2d7fc8637c46188d2aaf7c92d395cd50e737f39674d283decb17bcd9b7c45f5a0b592bf2fe3d48a8f121e89cdc0b5be35
-  data.tar.gz: 74a9dbe6e7a2082cf764db07058761ed2ca5fb36916e0f78d487b4648fd34b531ca801e2202706a138d45191ed8f0f7e05db357f400aa05569403eadfb8fc887
+  metadata.gz: b985c6f0b967354f3d1e5941ebeb2e5d6623b9d0abc33fd238f3764e72a6a6e10c293e7e61221d1be8ad89e631ecc7236bca597e809f1598ad54f0b305bb1103
+  data.tar.gz: 721f373b9eada6417b864e723504e5168df8f4362bef9edb4026aad2b39a3f99de5c72081f40ab80828d2f53d24c62c55e035ccd4964e3dadca2a27f3784e6e6

data/README.md

@@ -51,26 +51,26 @@ belonging to a lazy conversation:
 require "llm"
 
 llm = LLM.openai(ENV["KEY"])
-bot = llm.chat File.read("./share/llm/prompts/system.txt"), :system
-bot.chat "What color is the sky?"
-bot.chat "What color is an orange?"
-bot.chat "I like Ruby"
-bot.messages.each { print "[#{_1.role}] ", _1.content, "\n" }
+convo = llm.chat File.read("./share/llm/prompts/system.txt"), :system
+convo.chat "Tell me the answer to 5 + 15"
+convo.chat "Tell me the answer to (5 + 15) * 2"
+convo.chat "Tell me the answer to ((5 + 15) * 2) / 10"
+convo.messages.each { print "[#{_1.role}] ", _1.content, "\n" }
 
 ##
-# [system] You are a friendly chatbot. Sometimes, you like to tell a joke.
-#          But the joke must be based on the given inputs.
+# [system] You are my math assistant.
+#          I will provide you with (simple) equations.
+#          You will provide answers in the format "The answer to <equation> is <answer>".
 #          I will provide you a set of messages. Reply to all of them.
 #          A message is considered unanswered if there is no corresponding assistant response.
 #
-# [user] What color is the sky?
-# [user] What color is an orange?
-# [user] I like Ruby
+# [user] Tell me the answer to 5 + 15
+# [user] Tell me the answer to (5 + 15) * 2
+# [user] Tell me the answer to ((5 + 15) * 2) / 10
 #
-# [assistant] The sky is typically blue during the day. As for an orange,
-#             it is usually orange in color—funny how that works, right?
-#             I love Ruby too! Speaking of colors, why did the orange stop?
-#             Because it ran out of juice! 🍊😂
+# [assistant] The answer to 5 + 15 is 20.
+#             The answer to (5 + 15) * 2 is 40.
+#             The answer to ((5 + 15) * 2) / 10 is 4.
 ```
 
 #### Prompts
@@ -99,28 +99,87 @@ provider accepts:
 
 The
 [`LLM::Provider#embed`](https://0x1eef.github.io/x/llm/LLM/Provider.html#embed-instance_method)
-method generates a vector representation of a given piece of text.
-Embeddings capture the semantic meaning of text, and they are
-commonly used in tasks such as text similarity comparison (e.g., finding related documents),
-semantic search in vector databases, and the clustering and classification
-of text-based data:
+method generates a vector representation of one or more chunks
+of text. Embeddings capture the semantic meaning of text &ndash;
+a common use-case for them is to store chunks of text in a
+vector database, and then to query the database for *semantically
+similar* text. These chunks of similar text can then support the
+generation of a prompt that is used to query a large language model,
+which will go on to generate a response.
+
+For example, a user query might find similar text that adds important
+context to the prompt that informs the large language model in how to respond.
+The chunks of text may also carry metadata that can be used to further filter
+and contextualize the search results. This technique is popularly known as
+retrieval-augmented generation (RAG). Embeddings can also be used for
+other purposes as well &ndash; RAG is just one of the most popular use-cases.
+
+Let's take a look at an example that generates a couple of vectors
+for two chunks of text:
 
 ```ruby
 #!/usr/bin/env ruby
 require "llm"
 
 llm = LLM.openai(ENV["KEY"])
-res = llm.embed("Hello, world!")
+res = llm.embed(["programming is fun", "ruby is a programming language"])
 print res.class, "\n"
 print res.embeddings.size, "\n"
 print res.embeddings[0].size, "\n"
 
 ##
 # LLM::Response::Embedding
-# 1
+# 2
 # 1536
 ```
 
+### LLM
+
+#### Timeouts
+
+When running the ollama provider locally it might take a while for
+the language model to reply &ndash; depending on hardware and the
+size of the model. The following example demonstrates how to wait
+a longer period of time for a response through the use of the
+`timeout` configuration option with the `qwq` model. The following
+example waits up to 15 minutes for a response:
+
+```ruby
+#!/usr/bin/env ruby
+require "llm"
+
+llm = LLM.ollama(nil, timeout: 60*15)
+llm.chat "What is the meaning of life ?", model: "qwq"
+llm.last_message.tap { print "[assistant] ", _1.content, "\n" }
+```
+
+#### Models
+
+Generally each Large Language Model provides multiple models to choose
+from, and each model has its own set of capabilities and limitations.
+The following example demonstrates how to query the list of models
+through the
+[LLM::Provider#models](http://0x1eef.github.io/x/llm/LLM/Provider.html#models-instance_method)
+method &ndash; the example happens to use the ollama provider but
+this can be done for any provider:
+
+```ruby
+#!/usr/bin/env ruby
+require "llm"
+
+##
+# List models
+llm = LLM.ollama(nil)
+llm.models.each { print "#{_2.name}: #{_2.description}", "\n" }
+
+##
+# Select a model
+llm.chat "Hello, world!", model: llm.models["qwq"]
+
+##
+# This also works
+llm.chat "Hello, world!", model: "qwq"
+```
 ## Providers
 
 - [x] [Anthropic](https://www.anthropic.com/)
@@ -139,7 +198,9 @@ A complete API reference is available at [0x1eef.github.io/x/llm](https://0x1eef
 
 ## Install
 
-LLM has not been published to RubyGems.org yet. Stay tuned
+llm.rb can be installed via rubygems.org:
+
+	gem install llm.rb
 
 ## License
 

data/lib/llm/conversation.rb

@@ -20,8 +20,9 @@ module LLM
     ##
     # @param [LLM::Provider] provider
     #  A provider
-    def initialize(provider)
+    def initialize(provider, params = {})
       @provider = provider
+      @params = params
       @messages = []
     end
 
@@ -30,9 +31,20 @@ module LLM
     # @return [LLM::Conversation]
     def chat(prompt, role = :user, **params)
       tap do
-        completion = @provider.complete(prompt, role, **params.merge(messages:))
+        completion = @provider.complete(prompt, role, **@params.merge(params.merge(messages:)))
         @messages.concat [Message.new(role, prompt), completion.choices[0]]
       end
     end
+
+    ##
+    # @param [#to_s] role
+    #  The role of the last message.
+    #  Defaults to the LLM's assistant role (eg "assistant" or "model")
+    # @return [LLM::Message]
+    #  The last message for the given role
+    def last_message(role: @provider.assistant_role)
+      messages.reverse_each.find { _1.role == role.to_s }
+    end
+    alias_method :recent_message, :last_message
   end
 end

data/lib/llm/lazy_conversation.rb

@@ -24,8 +24,9 @@ module LLM
     ##
     # @param [LLM::Provider] provider
     #  A provider
-    def initialize(provider)
+    def initialize(provider, params = {})
       @provider = provider
+      @params = params
       @messages = LLM::MessageQueue.new(provider)
     end
 
@@ -33,7 +34,18 @@ module LLM
     # @param prompt (see LLM::Provider#prompt)
     # @return [LLM::Conversation]
     def chat(prompt, role = :user, **params)
-      tap { @messages << [prompt, role, params] }
+      tap { @messages << [prompt, role, @params.merge(params)] }
     end
+
+    ##
+    # @param [#to_s] role
+    #  The role of the last message.
+    #  Defaults to the LLM's assistant role (eg "assistant" or "model")
+    # @return [LLM::Message]
+    #  The last message for the given role
+    def last_message(role: @provider.assistant_role)
+      messages.reverse_each.find { _1.role == role.to_s }
+    end
+    alias_method :recent_message, :last_message
   end
 end

data/lib/llm/message.rb

@@ -20,7 +20,7 @@ module LLM
     # @param [Hash] extra
     # @return [LLM::Message]
     def initialize(role, content, extra = {})
-      @role = role
+      @role = role.to_s
       @content = content
       @extra = extra
     end

data/lib/llm/model.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class LLM::Model < Struct.new(:name, :parameters, :description, :to_param, keyword_init: true)
+  def to_json(*)
+    to_param.to_json(*)
+  end
+end

data/lib/llm/provider.rb

@@ -1,114 +1,133 @@
 # frozen_string_literal: true
 
-module LLM
-  require "llm/http_client"
+##
+# The Provider class represents an abstract class for
+# LLM (Language Model) providers
+class LLM::Provider
+  require_relative "http_client"
+  include LLM::HTTPClient
+
   ##
-  # The Provider class represents an abstract class for
-  # LLM (Language Model) providers
-  class Provider
-    include HTTPClient
-    ##
-    # @param [String] secret
-    #  The secret key for authentication
-    # @param [String] host
-    #  The host address of the LLM provider
-    # @param [Integer] port
-    #  The port number
-    def initialize(secret, host:, port: 443, ssl: true)
-      @secret = secret
-      @http = Net::HTTP.new(host, port).tap do |http|
-        http.use_ssl = ssl
-      end
+  # @param [String] secret
+  #  The secret key for authentication
+  # @param [String] host
+  #  The host address of the LLM provider
+  # @param [Integer] port
+  #  The port number
+  # @param [Integer] timeout
+  #  The number of seconds to wait for a response
+  def initialize(secret, host:, port: 443, timeout: 60, ssl: true)
+    @secret = secret
+    @http = Net::HTTP.new(host, port).tap do |http|
+      http.use_ssl = ssl
+      http.read_timeout = timeout
     end
+  end
 
-    ##
-    # Returns an inspection of the provider object
-    # @return [String]
-    # @note The secret key is redacted in inspect for security reasons
-    def inspect
-      "#<#{self.class.name}:0x#{object_id.to_s(16)} @secret=[REDACTED] @http=#{@http.inspect}>"
-    end
+  ##
+  # Returns an inspection of the provider object
+  # @return [String]
+  # @note The secret key is redacted in inspect for security reasons
+  def inspect
+    "#<#{self.class.name}:0x#{object_id.to_s(16)} @secret=[REDACTED] @http=#{@http.inspect}>"
+  end
 
-    ##
-    # @param [String] input
-    #  The input to embed
-    # @raise [NotImplementedError]
-    #  When the method is not implemented by a subclass
-    # @return [LLM::Response::Embedding]
-    def embed(input, **params)
-      raise NotImplementedError
-    end
+  ##
+  # @param [String, Array<String>] input
+  #  The input to embed
+  # @raise [NotImplementedError]
+  #  When the method is not implemented by a subclass
+  # @return [LLM::Response::Embedding]
+  def embed(input, **params)
+    raise NotImplementedError
+  end
 
-    ##
-    # Completes a given prompt using the LLM
-    # @param [String] prompt
-    #  The input prompt to be completed
-    # @param [Symbol] role
-    #  The role of the prompt (e.g. :user, :system)
-    # @raise [NotImplementedError]
-    #  When the method is not implemented by a subclass
-    # @return [LLM::Response::Completion]
-    def complete(prompt, role = :user, **params)
-      raise NotImplementedError
-    end
+  ##
+  # Completes a given prompt using the LLM
+  # @param [String] prompt
+  #  The input prompt to be completed
+  # @param [Symbol] role
+  #  The role of the prompt (e.g. :user, :system)
+  # @raise [NotImplementedError]
+  #  When the method is not implemented by a subclass
+  # @return [LLM::Response::Completion]
+  def complete(prompt, role = :user, **params)
+    raise NotImplementedError
+  end
 
-    ##
-    # Starts a new lazy conversation
-    # @param prompt (see LLM::Provider#complete)
-    # @param role (see LLM::Provider#complete)
-    # @raise (see LLM::Provider#complete)
-    # @return [LLM::LazyConversation]
-    def chat(prompt, role = :user, **params)
-      LazyConversation.new(self).chat(prompt, role, **params)
-    end
+  ##
+  # Starts a new lazy conversation
+  # @param prompt (see LLM::Provider#complete)
+  # @param role (see LLM::Provider#complete)
+  # @raise (see LLM::Provider#complete)
+  # @return [LLM::LazyConversation]
+  def chat(prompt, role = :user, **params)
+    LLM::LazyConversation.new(self, params).chat(prompt, role)
+  end
 
-    ##
-    # Starts a new conversation
-    # @param prompt (see LLM::Provider#complete)
-    # @param role (see LLM::Provider#complete)
-    # @raise (see LLM::Provider#complete)
-    # @return [LLM::Conversation]
-    def chat!(prompt, role = :user, **params)
-      Conversation.new(self).chat(prompt, role, **params)
-    end
+  ##
+  # Starts a new conversation
+  # @param prompt (see LLM::Provider#complete)
+  # @param role (see LLM::Provider#complete)
+  # @raise (see LLM::Provider#complete)
+  # @return [LLM::Conversation]
+  def chat!(prompt, role = :user, **params)
+    LLM::Conversation.new(self, params).chat(prompt, role)
+  end
 
-    private
+  ##
+  # @return [String]
+  #  Returns the role of the assistant in the conversation.
+  #  Usually "assistant" or "model"
+  def assistant_role
+    raise NotImplementedError
+  end
 
-    ##
-    # Prepares a request for authentication
-    # @param [Net::HTTP::Request] req
-    #  The request to prepare for authentication
-    # @raise [NotImplementedError]
-    #  (see LLM::Provider#complete)
-    def auth(req)
-      raise NotImplementedError
-    end
+  ##
+  # @return [Hash<String, LLM::Model>]
+  #  Returns a hash of available models
+  def models
+    raise NotImplementedError
+  end
 
-    ##
-    # @return [Module]
-    #  Returns the module responsible for parsing a successful LLM response
-    # @raise [NotImplementedError]
-    #  (see LLM::Provider#complete)
-    def response_parser
-      raise NotImplementedError
-    end
+  private
 
-    ##
-    # @return [Class]
-    #  Returns the class responsible for handling an unsuccessful LLM response
-    # @raise [NotImplementedError]
-    #  (see LLM::Provider#complete)
-    def error_handler
-      raise NotImplementedError
-    end
+  ##
+  # The headers to include with a request
+  # @raise [NotImplementedError]
+  #  (see LLM::Provider#complete)
+  def headers
+    raise NotImplementedError
+  end
 
-    ##
-    # Prepares a request before sending it
-    def preflight(req, body)
-      req.content_type = "application/json"
-      req.body = JSON.generate(body)
-      auth(req)
-      req
-    end
+  ##
+  # @return [Module]
+  #  Returns the module responsible for parsing a successful LLM response
+  # @raise [NotImplementedError]
+  #  (see LLM::Provider#complete)
+  def response_parser
+    raise NotImplementedError
+  end
+
+  ##
+  # @return [Class]
+  #  Returns the class responsible for handling an unsuccessful LLM response
+  # @raise [NotImplementedError]
+  #  (see LLM::Provider#complete)
+  def error_handler
+    raise NotImplementedError
+  end
+
+  ##
+  # @param [String] provider
+  #  The name of the provider
+  # @return [Hash<String, Hash>]
+  def load_models!(provider)
+    require "yaml" unless defined?(YAML)
+    rootdir  = File.realpath File.join(__dir__, "..", "..")
+    sharedir = File.join(rootdir, "share", "llm")
+    provider = provider.gsub(/[^a-z0-9]/i, "")
+    yaml     = File.join(sharedir, "models", "#{provider}.yml")
+    YAML.safe_load_file(yaml).transform_values { LLM::Model.new(_1) }
   end
 end

data/lib/llm/providers/anthropic/error_handler.rb

@@ -20,7 +20,7 @@ class LLM::Anthropic
     #  Raises a subclass of {LLM::Error LLM::Error}
     def raise_error!
       case res
-      when Net::HTTPForbidden
+      when Net::HTTPUnauthorized
         raise LLM::Error::Unauthorized.new { _1.response = res }, "Authentication error"
       when Net::HTTPTooManyRequests
         raise LLM::Error::RateLimit.new { _1.response = res }, "Too many requests"

data/lib/llm/providers/anthropic.rb

@@ -11,7 +11,6 @@ module LLM
     include Format
 
     HOST = "api.anthropic.com"
-    DEFAULT_PARAMS = {max_tokens: 1024, model: "claude-3-5-sonnet-20240620"}.freeze
 
     ##
     # @param secret (see LLM::Provider#initialize)
@@ -20,14 +19,17 @@ module LLM
     end
 
     ##
+    # Provides an embedding via VoyageAI per
+    # [Anthropic's recommendation](https://docs.anthropic.com/en/docs/build-with-claude/embeddings)
     # @param input (see LLM::Provider#embed)
+    # @param [String] token
+    #  Valid token for the VoyageAI API
+    # @param [Hash] params
+    #  Additional parameters to pass to the API
     # @return (see LLM::Provider#embed)
-    def embed(input, **params)
-      req = Net::HTTP::Post.new ["api.voyageai.com/v1", "embeddings"].join("/")
-      body = {input:, model: "voyage-2"}.merge!(params)
-      req = preflight(req, body)
-      res = request(@http, req)
-      Response::Embedding.new(res).extend(response_parser)
+    def embed(input, token:, **params)
+      llm = LLM.voyageai(token)
+      llm.embed(input, **params)
     end
 
     ##
@@ -36,20 +38,34 @@ module LLM
     # @param role (see LLM::Provider#complete)
     # @return (see LLM::Provider#complete)
     def complete(prompt, role = :user, **params)
-      req = Net::HTTP::Post.new ["/v1", "messages"].join("/")
+      params   = {max_tokens: 1024, model: "claude-3-5-sonnet-20240620"}.merge!(params)
+      req      = Net::HTTP::Post.new("/v1/messages", headers)
       messages = [*(params.delete(:messages) || []), Message.new(role, prompt)]
-      params = DEFAULT_PARAMS.merge(params)
-      body = {messages: format(messages)}.merge!(params)
-      req = preflight(req, body)
-      res = request(@http, req)
+      req.body = JSON.dump({messages: format(messages)}.merge!(params))
+      res      = request(@http, req)
       Response::Completion.new(res).extend(response_parser)
     end
 
+    ##
+    # @return (see LLM::Provider#assistant_role)
+    def assistant_role
+      "assistant"
+    end
+
+    ##
+    # @return (see LLM::Provider#models)
+    def models
+      @models ||= load_models!("anthropic")
+    end
+
     private
 
-    def auth(req)
-      req["anthropic-version"] = "2023-06-01"
-      req["x-api-key"] = @secret
+    def headers
+      {
+        "Content-Type" => "application/json",
+        "x-api-key" => @secret,
+        "anthropic-version" => "2023-06-01"
+      }
     end
 
     def response_parser

data/lib/llm/providers/gemini.rb

@@ -11,7 +11,6 @@ module LLM
     include Format
 
     HOST = "generativelanguage.googleapis.com"
-    DEFAULT_PARAMS = {model: "gemini-1.5-flash"}.freeze
 
     ##
     # @param secret (see LLM::Provider#initialize)
@@ -23,11 +22,10 @@ module LLM
     # @param input (see LLM::Provider#embed)
     # @return (see LLM::Provider#embed)
     def embed(input, **params)
-      path = ["/v1beta/models", "text-embedding-004"].join("/")
-      req = Net::HTTP::Post.new [path, "embedContent"].join(":")
-      body = {content: {parts: [{text: input}]}}
-      req = preflight(req, body)
-      res = request @http, req
+      path = ["/v1beta/models/text-embedding-004", "embedContent?key=#{@secret}"].join(":")
+      req = Net::HTTP::Post.new(path, headers)
+      req.body = JSON.dump({content: {parts: [{text: input}]}})
+      res = request(@http, req)
       Response::Embedding.new(res).extend(response_parser)
     end
 
@@ -37,20 +35,33 @@ module LLM
     # @param role (see LLM::Provider#complete)
     # @return (see LLM::Provider#complete)
     def complete(prompt, role = :user, **params)
-      params = DEFAULT_PARAMS.merge(params)
-      path = ["/v1beta/models", params.delete(:model)].join("/")
-      req = Net::HTTP::Post.new [path, "generateContent"].join(":")
+      params   = {model: "gemini-1.5-flash"}.merge!(params)
+      path     = ["/v1beta/models/#{params.delete(:model)}", "generateContent?key=#{@secret}"].join(":")
+      req      = Net::HTTP::Post.new(path, headers)
       messages = [*(params.delete(:messages) || []), LLM::Message.new(role, prompt)]
-      body = {contents: format(messages)}
-      req = preflight(req, body)
-      res = request(@http, req)
+      req.body = JSON.dump({contents: format(messages)})
+      res      = request(@http, req)
       Response::Completion.new(res).extend(response_parser)
     end
 
+    ##
+    # @return (see LLM::Provider#assistant_role)
+    def assistant_role
+      "model"
+    end
+
+    ##
+    # @return (see LLM::Provider#models)
+    def models
+      @models ||= load_models!("gemini")
+    end
+
     private
 
-    def auth(req)
-      req.path.replace [req.path, URI.encode_www_form(key: @secret)].join("?")
+    def headers
+      {
+        "Content-Type" => "application/json"
+      }
     end
 
     def response_parser