raix 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eaf67088dabee6158ede28a1a2279124257313326c9bea8ef3791efe3649e7ca
-  data.tar.gz: b35022cc748f4ab851044300ee57dcc5ee317d133bdbf68fb3aeb2a6cbaa4bcd
+  metadata.gz: 318057e8ece37b63c06884a61c37dc1ef15f38cee2c05d5305bcaf6d3697420e
+  data.tar.gz: 250229d71a808203689b87cea7c1ce8dd31b085c92b54dce392787299cd420d6
 SHA512:
-  metadata.gz: 8aa98c05854a5697de357f75bacb649e17f51be6a4fe183dfb227aedb803c01f8001ffed29948af2883f726950fd58706d93410abded42cf25e6139c5fde88a2
-  data.tar.gz: f7eda9fc1aaf073d874b9c9d3c3c243605d6799d7c0778f701ae53d8ea339e4ca11ed358a0b66894675d774cdd2561cbdf38471bcd577e3e2638f3db947db609
+  metadata.gz: dc51d8fab907f8ffa5e95df2ef308ee3cd5fc443f46803e8a6f184be80e12719be2d25aeb51349a7912b35df55866db66cdec532c0258bc66a97aeabbc4017d0
+  data.tar.gz: 7b393143a5da05ba75ac11a8a77e31b0e61c1e7e9cb564fc6c986a4f63e2f98a24d43056f613e8e4685baab85251c9e819bcf99fcdc22ee7428b50eb9895d4e7

data/.rubocop.yml

@@ -11,7 +11,7 @@ Style/StringLiteralsInInterpolation:
   EnforcedStyle: double_quotes
 
 Layout/LineLength:
-  Max: 120
+  Max: 180
 
 Metrics/BlockLength:
   Enabled: false

data/.ruby-version

@@ -0,0 +1 @@
+3.2.2

data/CHANGELOG.md

@@ -8,3 +8,9 @@
 - adds `ChatCompletion` module
 - adds `PromptDeclarations` module
 - adds `FunctionDispatch` module
+
+## [0.3.2] - 2024-06-29
+- adds support for streaming
+
+## [0.4.0] - 2024-10-18
+- adds support for Anthropic-style prompt caching

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    raix (0.1.0)
+    raix (0.3.2)
       activesupport (>= 6.0)
       open_router (~> 0.2)
 
@@ -41,6 +41,7 @@ GEM
     faraday-retry (2.2.1)
       faraday (~> 2.0)
     ffi (1.17.0-arm64-darwin)
+    ffi (1.17.0-x86_64-linux-gnu)
     formatador (1.1.0)
     guard (2.18.1)
       formatador (>= 0.2.4)
@@ -79,6 +80,8 @@ GEM
     netrc (0.11.0)
     nokogiri (1.16.6-arm64-darwin)
       racc (~> 1.4)
+    nokogiri (1.16.6-x86_64-linux)
+      racc (~> 1.4)
     notiffany (0.1.3)
       nenv (~> 0.1)
       shellany (~> 0.0)
@@ -164,6 +167,7 @@ GEM
       sorbet-static (= 0.5.11447)
     sorbet-runtime (0.5.11447)
     sorbet-static (0.5.11447-universal-darwin)
+    sorbet-static (0.5.11447-x86_64-linux)
     sorbet-static-and-runtime (0.5.11447)
       sorbet (= 0.5.11447)
       sorbet-runtime (= 0.5.11447)
@@ -194,6 +198,8 @@ GEM
 
 PLATFORMS
   arm64-darwin-21
+  arm64-darwin-22
+  x86_64-linux
 
 DEPENDENCIES
   activesupport (>= 6.0)

data/README.md

@@ -42,6 +42,30 @@ transcript << { role: "user", content: "What is the meaning of life?" }
 
 One of the advantages of OpenRouter and the reason that it is used by default by this library is that it handles mapping message formats from the OpenAI standard to whatever other model you're wanting to use (Anthropic, Cohere, etc.)
 
+### Prompt Caching
+
+Raix supports [Anthropic-style prompt caching](https://openrouter.ai/docs/prompt-caching#anthropic-claude) when using Anthropic's Claud family of models. You can specify a `cache_at` parameter when doing a chat completion. If the character count for the content of a particular message is longer than the cache_at parameter, it will be sent to Anthropic as a multipart message with a cache control "breakpoint" set to "ephemeral".
+
+Note that there is a limit of four breakpoints, and the cache will expire within five minutes. Therefore, it is recommended to reserve the cache breakpoints for large bodies of text, such as character cards, CSV data, RAG data, book chapters, etc. Raix does not enforce a limit on the number of breakpoints, which means that you might get an error if you try to cache too many messages.
+
+```ruby
+>> my_class.chat_completion(params: { cache_at: 1000 })
+=> {
+  "messages": [
+    {
+      "role": "system",
+      "content": [
+        {
+          "type": "text",
+          "text": "HUGE TEXT BODY LONGER THAN 1000 CHARACTERS",
+          "cache_control": {
+            "type": "ephemeral"
+          }
+        }
+      ]
+    },
+```
+
 ### Use of Tools/Functions
 
 The second (optional) module that you can add to your Ruby classes after `ChatCompletion` is `FunctionDispatch`. It lets you declare and implement functions to be called at the AI's discretion as part of a chat completion "loop" in a declarative, Rails-like "DSL" fashion.
@@ -216,6 +240,18 @@ If bundler is not being used to manage dependencies, install the gem by executin
 
     $ gem install raix
 
+If you are using the default OpenRouter API, Raix expects `Raix.configuration.openrouter_client` to initialized with the OpenRouter API client instance.
+
+You can add an initializer to your application's `config/initializers` directory:
+
+```ruby
+  # config/initializers/raix.rb
+  Raix.configure do |config|
+    config.openrouter_client = OpenRouter::Client.new
+  end
+```
+
+You will also need to configure the OpenRouter API access token as per the instructions here: https://github.com/OlympiaAI/open_router?tab=readme-ov-file#quickstart
 
 ## Development
 
@@ -235,4 +271,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Raix::Rails project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[OlympiaAI]/raix/blob/main/CODE_OF_CONDUCT.md).
+Everyone interacting in the Raix project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[OlympiaAI]/raix/blob/main/CODE_OF_CONDUCT.md).

data/lib/raix/chat_completion.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+require "active_support/core_ext/object/blank"
+require "raix/message_adapters/base"
+require "open_router"
+require "openai"
+
+module Raix
+  # The `ChatCompletion`` module is a Rails concern that provides a way to interact
+  # with the OpenRouter Chat Completion API via its client. The module includes a few
+  # methods that allow you to build a transcript of messages and then send them to
+  # the API for completion. The API will return a response that you can use however
+  # you see fit. If the response includes a function call, the module will dispatch
+  # the function call and return the result. Which implies that function calls need
+  # to be defined on the class that includes this module. (Note: You should probably
+  # use the `FunctionDispatch` module to define functions instead of doing it manually.)
+  module ChatCompletion
+    extend ActiveSupport::Concern
+
+    attr_accessor :cache_at, :frequency_penalty, :logit_bias, :logprobs, :loop, :min_p, :model, :presence_penalty,
+                  :repetition_penalty, :response_format, :stream, :temperature, :max_completion_tokens,
+                  :max_tokens, :seed, :stop, :top_a, :top_k, :top_logprobs, :top_p, :tools, :tool_choice, :provider
+
+    # This method performs chat completion based on the provided transcript and parameters.
+    #
+    # @param params [Hash] The parameters for chat completion.
+    # @option loop [Boolean] :loop (false) Whether to loop the chat completion after function calls.
+    # @option params [Boolean] :json (false) Whether to return the parse the response as a JSON object.
+    # @option params [Boolean] :openai (false) Whether to use OpenAI's API instead of OpenRouter's.
+    # @option params [Boolean] :raw (false) Whether to return the raw response or dig the text content.
+    # @return [String|Hash] The completed chat response.
+    def chat_completion(params: {}, loop: false, json: false, raw: false, openai: false)
+      # set params to default values if not provided
+      params[:cache_at] ||= cache_at.presence
+      params[:frequency_penalty] ||= frequency_penalty.presence
+      params[:logit_bias] ||= logit_bias.presence
+      params[:logprobs] ||= logprobs.presence
+      params[:max_completion_tokens] ||= max_completion_tokens.presence || Raix.configuration.max_completion_tokens
+      params[:max_tokens] ||= max_tokens.presence || Raix.configuration.max_tokens
+      params[:min_p] ||= min_p.presence
+      params[:presence_penalty] ||= presence_penalty.presence
+      params[:provider] ||= provider.presence
+      params[:repetition_penalty] ||= repetition_penalty.presence
+      params[:response_format] ||= response_format.presence
+      params[:seed] ||= seed.presence
+      params[:stop] ||= stop.presence
+      params[:temperature] ||= temperature.presence || Raix.configuration.temperature
+      params[:tool_choice] ||= tool_choice.presence
+      params[:tools] ||= tools.presence
+      params[:top_a] ||= top_a.presence
+      params[:top_k] ||= top_k.presence
+      params[:top_logprobs] ||= top_logprobs.presence
+      params[:top_p] ||= top_p.presence
+
+      if json
+        unless openai
+          params[:provider] ||= {}
+          params[:provider][:require_parameters] = true
+        end
+        params[:response_format] ||= {}
+        params[:response_format][:type] = "json_object"
+      end
+
+      # used by FunctionDispatch
+      self.loop = loop
+
+      # set the model to the default if not provided
+      self.model ||= Raix.configuration.model
+
+      adapter = MessageAdapters::Base.new(self)
+      messages = transcript.flatten.compact.map { |msg| adapter.transform(msg) }
+      raise "Can't complete an empty transcript" if messages.blank?
+
+      begin
+        response = if openai
+                     openai_request(params:, model: openai, messages:)
+                   else
+                     openrouter_request(params:, model:, messages:)
+                   end
+        retry_count = 0
+        content = nil
+
+        # no need for additional processing if streaming
+        return if stream && response.blank?
+
+        # tuck the full response into a thread local in case needed
+        Thread.current[:chat_completion_response] = response.with_indifferent_access
+
+        # TODO: add a standardized callback hook for usage events
+        # broadcast(:usage_event, usage_subject, self.class.name.to_s, response, premium?)
+
+        # TODO: handle parallel tool calls
+        if (function = response.dig("choices", 0, "message", "tool_calls", 0, "function"))
+          @current_function = function["name"]
+          # dispatch the called function
+          arguments = JSON.parse(function["arguments"].presence || "{}")
+          arguments[:bot_message] = bot_message if respond_to?(:bot_message)
+          return send(function["name"], arguments.with_indifferent_access)
+        end
+
+        response.tap do |res|
+          content = res.dig("choices", 0, "message", "content")
+          if json
+            content = content.squish
+            return JSON.parse(content)
+          end
+
+          return content unless raw
+        end
+      rescue JSON::ParserError => e
+        if e.message.include?("not a valid") # blank JSON
+          puts "Retrying blank JSON response... (#{retry_count} attempts) #{e.message}"
+          retry_count += 1
+          sleep 1 * retry_count # backoff
+          retry if retry_count < 3
+
+          raise e # just fail if we can't get content after 3 attempts
+        end
+
+        puts "Bad JSON received!!!!!!: #{content}"
+        raise e
+      rescue Faraday::BadRequestError => e
+        # make sure we see the actual error message on console or Honeybadger
+        puts "Chat completion failed!!!!!!!!!!!!!!!!: #{e.response[:body]}"
+        raise e
+      end
+    end
+
+    # This method returns the transcript array.
+    # Manually add your messages to it in the following abbreviated format
+    # before calling `chat_completion`.
+    #
+    # { system: "You are a pumpkin" },
+    # { user: "Hey what time is it?" },
+    # { assistant: "Sorry, pumpkins do not wear watches" }
+    #
+    # to add a function call use the following format:
+    # { function: { name: 'fancy_pants_function', arguments: { param: 'value' } } }
+    #
+    # to add a function result use the following format:
+    # { function: result, name: 'fancy_pants_function' }
+    #
+    # @return [Array] The transcript array.
+    def transcript
+      @transcript ||= []
+    end
+
+    private
+
+    def openai_request(params:, model:, messages:)
+      # deprecated in favor of max_completion_tokens
+      params.delete(:max_tokens)
+
+      params[:stream] ||= stream.presence
+      params[:stream_options] = { include_usage: true } if params[:stream]
+
+      params.delete(:temperature) if model == "o1-preview"
+
+      Raix.configuration.openai_client.chat(parameters: params.compact.merge(model:, messages:))
+    end
+
+    def openrouter_request(params:, model:, messages:)
+      # max_completion_tokens is not supported by OpenRouter
+      params.delete(:max_completion_tokens)
+
+      retry_count = 0
+
+      begin
+        Raix.configuration.openrouter_client.complete(messages, model:, extras: params.compact, stream:)
+      rescue OpenRouter::ServerError => e
+        if e.message.include?("retry")
+          puts "Retrying OpenRouter request... (#{retry_count} attempts) #{e.message}"
+          retry_count += 1
+          sleep 1 * retry_count # backoff
+          retry if retry_count < 5
+        end
+
+        raise e
+      end
+    end
+  end
+end

data/lib/raix/function_dispatch.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "securerandom"
+module Raix
+  # Provides declarative function definition for ChatCompletion classes.
+  #
+  # Example:
+  #
+  #   class MeaningOfLife
+  #     include Raix::ChatCompletion
+  #     include Raix::FunctionDispatch
+  #
+  #     function :ask_deep_thought do
+  #       wait 236_682_000_000_000
+  #       "The meaning of life is 42"
+  #     end
+  #
+  #     def initialize
+  #       transcript << { user: "What is the meaning of life?" }
+  #       chat_completion
+  #     end
+  #   end
+  module FunctionDispatch
+    extend ActiveSupport::Concern
+
+    class_methods do
+      attr_reader :functions
+
+      # Defines a function that can be dispatched by the ChatCompletion module while
+      # processing the response from an AI model.
+      #
+      # Declaring a function here will automatically add it (in JSON Schema format) to
+      # the list of tools provided to the OpenRouter Chat Completion API. The function
+      # will be dispatched by name, so make sure the name is unique. The function's block
+      # argument will be executed in the instance context of the class that includes this module.
+      #
+      # Example:
+      #   function :google_search, "Search Google for something", query: { type: "string" } do |arguments|
+      #     GoogleSearch.new(arguments[:query]).search
+      #   end
+      #
+      # @param name [Symbol] The name of the function.
+      # @param description [String] An optional description of the function.
+      # @param parameters [Hash] The parameters that the function accepts.
+      # @param block [Proc] The block of code to execute when the function is called.
+      def function(name, description = nil, **parameters, &block)
+        @functions ||= []
+        @functions << begin
+          { name:, parameters: { type: "object", properties: {} } }.tap do |definition|
+            definition[:description] = description if description.present?
+            parameters.map do |key, value|
+              definition[:parameters][:properties][key] = value
+            end
+          end
+        end
+
+        define_method(name) do |arguments|
+          id = SecureRandom.uuid[0, 23]
+          transcript << {
+            role: "assistant",
+            content: nil,
+            tool_calls: [
+              {
+                id:,
+                type: "function",
+                function: {
+                  name:,
+                  arguments: arguments.to_json
+                }
+              }
+            ]
+          }
+          instance_exec(arguments, &block).tap do |content|
+            transcript << {
+              role: "tool",
+              tool_call_id: id,
+              name:,
+              content: content.to_s
+            }
+            # TODO: add on_error handler as optional parameter to function
+          end
+
+          chat_completion(**chat_completion_args) if loop
+        end
+      end
+    end
+
+    included do
+      attr_accessor :chat_completion_args
+    end
+
+    def chat_completion(**chat_completion_args)
+      raise "No functions defined" if self.class.functions.blank?
+
+      self.chat_completion_args = chat_completion_args
+
+      super
+    end
+
+    # Stops the looping of chat completion after function calls.
+    # Useful for manually halting processing in workflow components
+    # that do not require a final text response to an end user.
+    def stop_looping!
+      self.loop = false
+    end
+
+    def tools
+      self.class.functions.map { |function| { type: "function", function: } }
+    end
+  end
+end

data/lib/raix/prompt_declarations.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "ostruct"
+
+module Raix
+  # The PromptDeclarations module provides a way to chain prompts and handle
+  # user responses in a serialized manner (in the order they were defined),
+  # with support for functions if the FunctionDispatch module is also included.
+  module PromptDeclarations
+    extend ActiveSupport::Concern
+    extend ChatCompletion
+
+    module ClassMethods # rubocop:disable Style/Documentation
+      # Adds a prompt to the list of prompts.
+      #
+      # @param system [Proc] A lambda that generates the system message.
+      # @param text [Proc] A lambda that generates the prompt text. (Required)
+      # @param success [Proc] The block of code to execute when the prompt is answered.
+      # @param parameters [Hash] Additional parameters for the completion API call
+      # @param stream [Boolean] Whether to stream the response.
+      def prompt(text:, system: nil, success: nil, params: {}, stream: false)
+        name = Digest::SHA256.hexdigest(text.inspect)[0..7]
+        prompts << begin
+          OpenStruct.new({ name:, system:, text:, success:, params:, stream: })
+        end
+
+        define_method(name) do |response|
+          if Rails.env.local?
+            puts "_" * 80
+            puts "PromptDeclarations#response:"
+            puts "#{text.source_location} (#{name})"
+            puts response
+            puts "_" * 80
+          end
+
+          return response if success.nil?
+          return send(success, response) if success.is_a?(Symbol)
+
+          instance_exec(response, &success)
+        end
+      end
+
+      # the list of prompts declared at class level
+      def prompts
+        @prompts ||= []
+      end
+
+      # getter/setter for system prompt declared at class level
+      def system_prompt(prompt = nil)
+        prompt ? @system_prompt = prompt.squish : @system_prompt
+      end
+    end
+
+    # Executes the chat completion process based on the class-level declared prompts.
+    # The response to each prompt is added to the transcript automatically and returned.
+    #
+    # Prompts require at least a `text` lambda parameter.
+    #
+    # @param params [Hash] Parameters for the chat completion override those defined in the current prompt.
+    # @option params [Boolean] :raw (false) Whether to return the raw response or dig the text content.
+    #
+    # Uses system prompt in following order of priority:
+    #   - system lambda specified in the prompt declaration
+    #   - system_prompt instance method if defined
+    #   - system_prompt class-level declaration if defined
+    #
+    #  TODO: shortcut syntax passes just a string prompt if no other options are needed.
+    #
+    # @raise [RuntimeError] If no prompts are defined.
+    #
+    def chat_completion(params: {}, raw: false)
+      raise "No prompts defined" unless self.class.prompts.present?
+
+      current_prompts = self.class.prompts.clone
+
+      while (@current_prompt = current_prompts.shift)
+        __system_prompt = instance_exec(&@current_prompt.system) if @current_prompt.system.present? # rubocop:disable Lint/UnderscorePrefixedVariableName
+        __system_prompt ||= system_prompt if respond_to?(:system_prompt)
+        __system_prompt ||= self.class.system_prompt.presence
+        transcript << { system: __system_prompt } if __system_prompt
+        transcript << { user: instance_exec(&@current_prompt.text) } # text is required
+
+        params = @current_prompt.params.merge(params)
+
+        # set the stream if necessary
+        self.stream = instance_exec(&@current_prompt.stream) if @current_prompt.stream.present?
+
+        super(params:, raw:).then do |response|
+          transcript << { assistant: response }
+          @last_response = send(@current_prompt.name, response)
+        end
+      end
+
+      @last_response
+    end
+
+    # Returns the model parameter of the current prompt or the default model.
+    #
+    # @return [Object] The model parameter of the current prompt or the default model.
+    def model
+      @current_prompt.params[:model] || super
+    end
+
+    # Returns the temperature parameter of the current prompt or the default temperature.
+    #
+    # @return [Float] The temperature parameter of the current prompt or the default temperature.
+    def temperature
+      @current_prompt.params[:temperature] || super
+    end
+
+    # Returns the max_tokens parameter of the current prompt or the default max_tokens.
+    #
+    # @return [Integer] The max_tokens parameter of the current prompt or the default max_tokens.
+    def max_tokens
+      @current_prompt.params[:max_tokens] || super
+    end
+  end
+end

data/lib/raix/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Raix
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
 end

data/lib/raix.rb

@@ -16,6 +16,9 @@ module Raix
     # The max_tokens option determines the maximum number of tokens to generate.
     attr_accessor :max_tokens
 
+    # The max_completion_tokens option determines the maximum number of tokens to generate.
+    attr_accessor :max_completion_tokens
+
     # The model option determines the model to use for text generation. This option
     # is normally set in each class that includes the ChatCompletion module.
     attr_accessor :model
@@ -27,12 +30,14 @@ module Raix
     attr_accessor :openai_client
 
     DEFAULT_MAX_TOKENS = 1000
+    DEFAULT_MAX_COMPLETION_TOKENS = 16_384
     DEFAULT_MODEL = "meta-llama/llama-3-8b-instruct:free"
     DEFAULT_TEMPERATURE = 0.0
 
     # Initializes a new instance of the Configuration class with default values.
     def initialize
       self.temperature = DEFAULT_TEMPERATURE
+      self.max_completion_tokens = DEFAULT_MAX_COMPLETION_TOKENS
       self.max_tokens = DEFAULT_MAX_TOKENS
       self.model = DEFAULT_MODEL
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: raix
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Obie Fernandez
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-06-26 00:00:00.000000000 Z
+date: 2024-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -47,6 +47,7 @@ extra_rdoc_files: []
 files:
 - ".rspec"
 - ".rubocop.yml"
+- ".ruby-version"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -55,6 +56,9 @@ files:
 - README.md
 - Rakefile
 - lib/raix.rb
+- lib/raix/chat_completion.rb
+- lib/raix/function_dispatch.rb
+- lib/raix/prompt_declarations.rb
 - lib/raix/version.rb
 - raix.gemspec
 - sig/raix.rbs
@@ -80,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.21
 signing_key:
 specification_version: 4
 summary: Ruby AI eXtensions