glim_ai 0.2.0

data/lib/anthropic_response.rb

@@ -0,0 +1,101 @@
+require 'anthropic'
+#require_relative 'globals'
+#require 'tiktoken_ruby' # TODO only for token counting while anthropic doesn't support it
+
+class AnthropicResponse < GlimResponse
+
+  def self._llm_info(llm_name)
+    if llm_name.start_with?("claude-instant-1")
+      prompt_price = 1.63
+      completion_price = 5.51
+      context_length = 100_000
+    elsif llm_name.start_with?("claude-2")
+      prompt_price =  11.02
+      completion_price = 32.68
+      context_length = 100_000
+    else
+      raise "Unknown model #{llm_name}"
+    end
+    cost_per_prompt_token = prompt_price / 1_000_000.0
+    cost_per_completion_token = completion_price / 1_000_000.0
+    return {cost_per_prompt_token:,cost_per_completion_token:,context_length:}
+  end
+
+  def self._count_tokens(model, s)
+    # TODO -- not yet support by ruby-antrhopic
+    # client = Anthropic::Client.new
+    # puts "***** #{client.count_tokens(req.prompt)}"
+    
+    enc = Tiktoken.encoding_for_model("gpt-3.5-turbo") # this is obviously wrong, should use anthropic
+    return enc.encode(s).length
+  end
+
+  # Anthropic does not report token counts so we use the number from the request
+  def prompt_tokens
+    req.prompt_token_count
+  end
+
+  # Anthropic does not report token counts so we count ourselves
+  # NOTE: This means there may be some inaccuracies.
+  def completion_tokens
+    wait_for_response
+    req.count_tokens(completion)
+  end
+
+  def responding_llm_name
+    "TODO FOR ANTHROPIC"
+  end
+  
+  def self.api_limiter
+    @_api_limiter ||= APILimiter.new(max_concurrent_requests: 1)
+  end
+
+  attr_reader :total_token_count, :prompt_token_count, :completion_token_count
+
+  def process_response_from_api
+    @completion = raw_response[:completion] || err("Anthropic API error: No completion in #{raw_response}")
+    # puts "JSON: #{JSON.pretty_generate(raw_response)}"
+    usage = raw_response[:usage]
+    if usage
+      @total_token_count = usage[:total_tokens] || err("No total_tokens in #{usage}")
+      @prompt_token_count = usage[:prompt_tokens] || err("No prompt_tokens in #{usage}")
+      @completion_token_count = usage[:completion_tokens] || err("No completion_tokens in #{usage}")
+    else
+      #raise err("No usage in #{raw_response}")
+    end
+
+  end
+
+  def async_send_request_to_api
+    # anthropic API modifies the prompt
+    # but no need to params = Marshal.load(Marshal.dump(req.request_hash))
+    # since it's only the prompt, we can just do this:
+    params = req.request_hash.dup
+    params[:prompt] = req.request_hash[:prompt].dup
+
+    raise "request_hash should not have messages for Anthropic" if params[:messages]
+    @thread = Thread.new do
+      AnthropicResponse.api_limiter.with_limit do
+        client = Anthropic::Client.new
+        # this is necessary because the Anthropic API modifies the prompt
+        _raw_response = client.complete(parameters: params).with_indifferent_access
+        if _raw_response[:error]
+          if _raw_response[:error][:type] == "rate_limit_error"
+            limit = AnthropicResponse.api_limiter.max_concurrent_requests
+            raise RateLimitExceededError, "Rate limit (#{limit}) exceeded. Edit config or negotiate with Anthropic to avoid this."
+          else
+              "Anthropic API error: #{_raw_response[:error]}"
+          end
+        end
+        _raw_response
+      end
+    end
+  end
+end
+  
+  ##### NOT SUPPORTED in the code yet
+  # top_k
+  # integer
+  # Only sample from the top K options for each subsequent token.
+  # Used to remove "long tail" low probability responses. Learn more technical details here.
+  

data/lib/chat_request_details.rb

@@ -0,0 +1,140 @@
+require 'json-schema'
+require 'net/http'
+
+require 'openai'
+require 'tiktoken_ruby'
+
+require_relative 'globals'
+
+
+
+# GlimRequest delegates to this
+class ChatRequestDetails  # only for requests that involve a message array, like OpenAI  
+
+  def initialize(req)
+    @req = req
+  end
+  attr_accessor :req
+
+  def response_class
+    ChatResponse
+  end
+
+  def llm_class_changed
+    update_request_hash
+  end
+
+  def update_request_hash
+    req.request_hash[:temperature] = req.temperature / 2.0 if req.temperature # for some reason, OpenAI goes from 0 to 2, not 0 to 1
+    req.request_hash[:max_tokens] = req.max_tokens
+    req.request_hash[:model] = req.llm_name 
+    
+    #this is risky because it can overwrite things....
+
+    if @message_history
+        messages = req.message_history.dup
+    else
+      messages = [{"role":"system","content":"You are a helpful assistant."}] 
+    end
+    # this could make sense, for example, if message_history
+    # has a function call in it, along with the response to the function call that glim inserted
+    # TODO: we might want to handle that case differently.
+    if req.prompt
+      messages.append({ role: 'user', content: req.prompt }) 
+    end
+
+    req.request_hash[:messages] = messages
+
+  end
+
+  def set_message_history(messages)
+    @message_history = messages
+    update_request_hash
+  end
+  attr_reader :message_history
+
+  def replace_initial_system_message(system_message)
+    @message_history ||= []
+    @message_history[0] ||=  {
+      "role": "system",
+      "content": system_message
+    }
+  end
+
+
+  # for convenience, if you want a list, you can specify the schema for the items
+  def set_output_schema(output_schema, *flags)
+    output_schema.must_be_a Hash
+    meta_schema = JSON::Validator.validator_for_name('draft4').metaschema
+    begin
+      JSON::Validator.validate!(meta_schema, output_schema)
+      # putt :extract_data, 'The schema is valid.'        
+      @output_schema = output_schema
+    rescue JSON::Schema::ValidationError => e
+      putt :extract_data, "The schema is not valid. Reason: #{e.message}"
+      putt :extract_data, "Schema: #{output_schema}"
+      raise
+    end
+
+    if flags.include?(:list)
+      @output_schema = {
+        type: 'object',
+        properties: {
+          list: {
+            type: 'array',
+            items: output_schema
+          }
+        }
+      }
+    end
+  
+    extract_data_function_name = 'extract_data'
+    req.request_hash[:functions] = [{
+      name: extract_data_function_name,
+      description: "Extracts data from the user's message",
+      parameters: @output_schema
+    }]
+    # Specifying a particular function via {"name":\ "my_function"} forces the model to call that function.
+    req.request_hash[:function_call] = { "name": extract_data_function_name }
+  end
+  attr_reader :output_schema
+
+  def expected_output_is_list?
+    @output_schema && @output_schema[:type] == 'object' && @output_schema[:properties] && @output_schema[:properties][:list]
+  end
+
+  # function that in this request we offer to
+  # the LLM API to call
+  def set_functions_object(functions_object)
+    @functions_object = functions_object
+    # update_request_hash
+    req.request_hash[:functions] = functions_object.class.ai_method_signatures_clean
+    # [{
+    #   name: extract_data_function_name,
+    #   description: "Extracts data from the user's message",
+    #   parameters: @output_schema
+    # }]
+    # # Specifying a particular function via {"name":\ "my_function"} forces the model to call that function.
+    # request_data[:function_call] = { "name": extract_data_function_name }
+  end
+
+  def force_function_call(function_name)
+    req.request_hash[:function_call] = { "name": function_name }
+  end
+
+  attr_reader :functions_object
+
+  def to_s
+    s = "ChatRequestDetails: "
+    if output_schema
+      schema = JSON.pretty_generate(output_schema)
+      s += "\nSchema:\n#{schema}\n"
+    end
+    if functions_object
+      s += "\nFunctions object: #{functions_object}"
+    end
+    return s
+  end
+
+end
+

data/lib/chat_response.rb

@@ -0,0 +1,303 @@
+require 'json-schema'
+require 'net/http'
+
+require 'openai'
+require 'tiktoken_ruby'
+
+require_relative 'glim_ai_callable'
+
+require_relative 'globals'
+
+OpenAI.configure do |config|
+  config.access_token = ENV.fetch('OPENAI_API_KEY')
+  config.request_timeout = 480 # Optional
+end
+
+
+class ChatResponse < GlimResponse
+
+  def self._llm_info(model)
+    if model.start_with?("gpt-3.5-turbo-16k")
+      prompt_price = 3.0 
+      completion_price = prompt_price / 3.0 * 4.0
+      context_length = 16384
+    elsif model.start_with?("gpt-3.5")
+      prompt_price =  1.5
+      completion_price = prompt_price / 3.0 * 4.0
+      context_length = 4096
+    elsif model.start_with?("gpt-4-32k")
+      prompt_price = 60.0
+      completion_price = 2 * prompt_price
+      context_length = 32768
+    elsif model.start_with?("gpt-4")
+      prompt_price = 30.0
+      completion_price = 2 * prompt_price
+      context_length = 8192
+    elsif model.start_with?("meta-llama")
+      prompt_price = 1.0
+      completion_price = prompt_price
+      context_length = 4096
+    elsif model.start_with?("codellama")
+      prompt_price = 1.0
+      completion_price = prompt_price
+      context_length = 4096
+    else
+      raise "Unknown model #{model}"
+    end
+    cost_per_prompt_token = prompt_price / 1_000_000.0
+    cost_per_completion_token = completion_price / 1_000_000.0
+    return {cost_per_prompt_token:,cost_per_completion_token:,context_length:}
+  end
+
+  def self._count_tokens(llm_name, s)
+    enc = Tiktoken.encoding_for_model(llm_name)
+    if !enc
+      # putt :warning, "Tiktoken doesn't know model #{llm_name}"
+      enc = Tiktoken.encoding_for_model("gpt-3.5-turbo")
+    end
+    return enc.encode(s).length
+  end
+
+  def self.api_limiter
+    @_api_limiter ||= APILimiter.new(max_concurrent_requests: 2)
+  end
+
+    # TODO -- some day we'll want to support tokens
+    # token_string = params.to_s
+    # enc = Tiktoken.encoding_for_model("gpt-4")
+    # @token_estimate = enc.encode(token_string).length
+
+
+  def placeholder_anyscale_api_call(params)
+    # for some reason the ruby gem for OpenAI won't work with anyscale
+    # so we're doing it manually for now
+    key = ENV.fetch('ANYSCALE_API_KEY')
+    # for some reason, this doesn't work, just returns "details: Not found"
+    # client = OpenAI::Client.new(uri_base: "https://api.endpoints.anyscale.com/v1", access_token: key)
+    # @raw_response = deep_symbolize_keys(cached_response || client.chat(parameters: params))
+    api_base = "https://api.endpoints.anyscale.com/v1"
+    uri = URI("#{api_base}/chat/completions")
+    r = Net::HTTP::Post.new(uri, 'Content-Type' => 'application/json', 'Authorization' => "Bearer #{key}")
+    r.body = params.to_json
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.read_timeout = 480  # seconds
+    http.use_ssl = true if uri.scheme == 'https'
+    response = http.request(r)
+    return JSON.parse(response.body).with_indifferent_access
+  end
+
+  def async_send_request_to_api
+    h = req.request_hash
+    log_request_hash
+    
+    raise "model not set; remember to set request.llm_name = (the model you want to use)" unless h[:model]
+
+    @thread = Thread.new do
+      ChatResponse.api_limiter.with_limit do
+        if GlimRequest.openai_llms.include?(req.llm_name)
+          client = OpenAI::Client.new
+          _raw_response = client.chat(parameters: req.request_hash).with_indifferent_access
+        else
+          _raw_response = placeholder_anyscale_api_call(req.request_hash).with_indifferent_access
+        end
+        if _raw_response[:error]
+          if _raw_response[:error][:type] == "rate_limit_error"
+            limit = self.api_limiter.max_concurrent_requests
+            raise RateLimitExceededError, "Rate limit (#{limit}) exceeded. Edit config or negotiate with Anthropic to avoid this."
+          else
+            puts "-------Error. Prompt:"
+            puts req.prompt.class
+            raise "OpenAI API error: #{_raw_response[:error]}"
+          end
+        end
+        _raw_response
+      end
+    end # thread; its value is the raw_response
+  end
+
+
+  def process_response_from_api
+    raise "no raw response!" unless @raw_response
+
+    # TODO
+    # request has message_history
+    # response has messages_sent (still need reader)
+    # and then what does the following look like? 
+
+    @message = (raw_response.dig(:choices, 0, :message) || err("No message returned!"))
+    @messages = messages_sent.dup + [@message]
+    @completion = @message[:content] # turns out there is always a completion, even with function calls
+    log_completion
+    if function_call_message?
+      log_function_call_message
+    end
+
+  end
+
+  def function_call_message?
+    wait_for_response
+    @message[:function_call] != nil
+  end
+
+  def _function_call_from_message
+    wait_for_response
+    @message[:function_call] || err("No function call!")
+  end
+
+  def _function_call_arguments_from_message
+    begin 
+      s = _function_call_from_message[:arguments]
+      JSON.parse(s).with_indifferent_access
+    rescue   => e
+      puts "JSON parse error:"
+      puts s
+      log_json_error(s)
+      err(e)
+    end
+  end
+
+  def function_name_from_message
+    _function_call_from_message[:name]
+  end
+
+  def log_json_error(s)
+    req.save_log_file("json_error.json", s)
+  end
+
+  def log_function_call_message
+    s = "LLM requested results of function call to #{req.functions_object}##{function_name_from_message}\n"
+    s += JSON.pretty_generate(_function_call_arguments_from_message)
+    req.save_log_file("function_call.txt", s)
+  end
+
+  # returns a new GlimRequest that is preloaded with the data for
+  # sending the results of a function call back to the LLM API
+  def create_request_with_function_result
+    wait_for_response
+
+    eval_functions_object = req.functions_object || err("No functions_object")
+    raise "functions_object must be ai_callable, is #{eval_functions_object}" unless eval_functions_object.is_a?(AICallable)
+
+    eval_function_name = _function_call_from_message[:name].to_sym
+    raise "no_method_error #{eval_function_name}" unless eval_functions_object.respond_to?(eval_function_name)
+
+    # TODO -- validate that the schema is right? 
+    eval_function_arguments = _function_call_arguments_from_message
+
+    putt :functions, "#{eval_functions_object}.#{eval_function_name}(#{eval_function_arguments})"
+    eval_function_result = eval_functions_object._perform_ai_call(eval_function_name, eval_function_arguments)
+
+    return create_request_for_chat(message_to_append: {
+      role: "function",
+      name: eval_function_name,
+      content: eval_function_result.to_json
+    })
+  end
+
+  def create_request_for_chat(message_to_append: nil)
+    wait_for_response
+    h = req.generic_params_hash.merge({
+      llm_name: req.llm_name,
+      context: req.context
+    })
+    new_request = GlimRequest.new(**h)
+    new_request.set_message_history(messages.dup)
+
+    if message_to_append
+      messages.append(message_to_append)
+    end
+
+    new_request.request_hash[:messages] = messages
+    new_request
+
+  end
+
+  # the message generated by GPT
+  def message
+    wait_for_response
+    @message 
+  end
+
+  # all messages: prior ones, the prompt, GPT's response, and the function call, if it happened
+  def messages
+    wait_for_response
+    @messages
+  end
+
+  # the extracted data generated by GPT
+  def extracted_data
+    return @extracted_data if @extracted_data
+    raise "no output schema specified, can't get extracted_data" unless req.output_schema
+    wait_for_response
+    args = _function_call_arguments_from_message
+
+    JSON::Validator.validate!(req.output_schema, args)
+    if req.expected_output_is_list? # TODO -- this feels a bit awkward
+      @extracted_data = args[:list] || raise(err("Expected list"))
+    else
+      @extracted_data = args
+    end
+    @extracted_data
+  end
+
+  def messages_sent
+    req.request_hash[:messages] 
+  end
+
+  def usage
+    raw_response[:usage]
+  end
+
+  def prompt_tokens
+    usage[:prompt_tokens]
+  end
+
+  def completion_tokens
+    usage[:completion_tokens]
+  end
+
+  def responding_llm_name
+    raw_response[:model]
+  end
+
+end
+
+
+
+######## OpenAI only - TODO
+# maybe put this in a hash called ai.open_ai ?
+
+# n
+# integer or null
+# Optional
+# Defaults to 1
+# How many chat completion choices to generate for each input message.
+
+#-------
+
+# presence_penalty
+# number or null
+# Optional
+# Defaults to 0
+# Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.
+# See more information about frequency and presence penalties.
+
+# frequency_penalty
+# number or null
+# Optional
+# Defaults to 0
+# Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.
+# See more information about frequency and presence penalties.
+
+# logit_bias
+# map
+# Optional
+# Defaults to null
+# Modify the likelihood of specified tokens appearing in the completion.
+# Accepts a json object that maps tokens (specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically, the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model, but values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should result in a ban or exclusive selection of the relevant token.
+
+
+      # "none" means the model does not call a function, and responds to the end-user.
+      # "auto" means the model can pick between an end-user or calling a function.
+      # Specifying a particular function via {"name":\ "my_function"} forces the model to call that function.
+      # "none" is the default when no functions are present. "auto" is the default if functions are present.

data/lib/glim_ai/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module GlimAI
+  VERSION = '0.2.0'.freeze
+end

data/lib/glim_ai.rb

@@ -0,0 +1,8 @@
+require_relative 'glim_ai/version'
+
+require_relative 'glim_context'
+# Include other requires as needed, e.g., 'glim_request', 'glim_response'
+
+module GlimAI
+  # Additional constants or methods if needed
+end