eval-ruby 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 07e82b10ad871e882d8a5da7a3d85ef7436c3b1695840c2974b22a0df70ae0fc
-  data.tar.gz: f483be5c375db41ff367162257186dde52627aa4b9bba4d493dfa1455363d310
+  metadata.gz: 59b7bd64cf696d82a27cb6330ab948306904f444a8a64476f702ee1937bbccab
+  data.tar.gz: f9c0c234f0712d37d309d460d1c3204d3e6f70bfbd8fe4dcff95ac508bcb7f34
 SHA512:
-  metadata.gz: b4938e44301b2440500d6506057588fbafa5ca91f6cf574f288690132748ba81f2010c61ba76ad2f536a5d3cb7610442f1db854b08f0a8bf65d4bef7cf3b607c
-  data.tar.gz: f76fbe015937d962fb9747ff90a0fc245ae32fc47834644a894d4b69595dd31c30cd621922dca94386e8e737c6be82f726e4b76d039ebd38b8f73d6f548b5ce7
+  metadata.gz: 6a9f0c12a790b0098ba639bc236c6080c64c7fbb4ad9892c36810e93b88486bebaa42dca9245beaf828beca29e31793030d62ab232cad50107bc99905635a069
+  data.tar.gz: 7c922b6fd8743d5241a254baf301aae2af6aba936fdeedbcb467f9549a8f16493c7571ec9724a5460a75228e415cbcaa852b915ef78f23aa8f1fcbddb85d9450

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    eval-ruby (0.1.0)
+    eval-ruby (0.1.1)
       csv
 
 GEM
@@ -39,7 +39,7 @@ CHECKSUMS
   bigdecimal (4.0.1) sha256=8b07d3d065a9f921c80ceaea7c9d4ae596697295b584c296fe599dd0ad01c4a7
   crack (1.0.1) sha256=ff4a10390cd31d66440b7524eb1841874db86201d5b70032028553130b6d4c7e
   csv (3.3.5) sha256=6e5134ac3383ef728b7f02725d9872934f523cb40b961479f69cf3afa6c8e73f
-  eval-ruby (0.1.0)
+  eval-ruby (0.1.1)
   hashdiff (1.2.1) sha256=9c079dbc513dfc8833ab59c0c2d8f230fa28499cc5efb4b8dd276cf931457cd1
   minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5
   public_suffix (7.0.5) sha256=1a8bb08f1bbea19228d3bed6e5ed908d1cb4f7c2726d18bd9cadf60bc676f623

data/lib/eval_ruby/dataset.rb

@@ -105,7 +105,7 @@ module EvalRuby
         PROMPT
 
         result = judge.call(prompt)
-        next unless result&.key?("pairs")
+        next unless result.is_a?(Hash) && result.key?("pairs")
 
         result["pairs"].each do |pair|
           dataset.add(

data/lib/eval_ruby/judges/anthropic.rb

@@ -9,31 +9,46 @@ module EvalRuby
     class Anthropic < Base
       API_URL = "https://api.anthropic.com/v1/messages"
 
+      def initialize(config)
+        super
+        raise EvalRuby::Error, "API key is required. Set via EvalRuby.configure { |c| c.api_key = '...' }" if @config.api_key.nil? || @config.api_key.empty?
+      end
+
       def call(prompt)
-        uri = URI(API_URL)
-        request = Net::HTTP::Post.new(uri)
-        request["x-api-key"] = @config.api_key
-        request["anthropic-version"] = "2023-06-01"
-        request["Content-Type"] = "application/json"
-        request.body = JSON.generate({
-          model: @config.judge_model,
-          max_tokens: 4096,
-          messages: [{role: "user", content: prompt}],
-          temperature: 0.0
-        })
+        retries = 0
+        begin
+          uri = URI(API_URL)
+          request = Net::HTTP::Post.new(uri)
+          request["x-api-key"] = @config.api_key
+          request["anthropic-version"] = "2023-06-01"
+          request["Content-Type"] = "application/json"
+          request.body = JSON.generate({
+            model: @config.judge_model,
+            max_tokens: 4096,
+            messages: [{role: "user", content: prompt}],
+            temperature: 0.0
+          })
 
-        response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true,
-                                   read_timeout: @config.timeout) do |http|
-          http.request(request)
-        end
+          response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true,
+                                     read_timeout: @config.timeout) do |http|
+            http.request(request)
+          end
 
-        unless response.is_a?(Net::HTTPSuccess)
-          raise Error, "Anthropic API error: #{response.code} - #{response.body}"
-        end
+          unless response.is_a?(Net::HTTPSuccess)
+            raise Error, "Anthropic API error: #{response.code} - #{response.body}"
+          end
 
-        body = JSON.parse(response.body)
-        content = body.dig("content", 0, "text")
-        parse_json_response(content)
+          body = JSON.parse(response.body)
+          content = body.dig("content", 0, "text")
+          parse_json_response(content)
+        rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNRESET => e
+          retries += 1
+          if retries <= @config.max_retries
+            sleep(2 ** (retries - 1))
+            retry
+          end
+          raise EvalRuby::TimeoutError, "Judge API failed after #{@config.max_retries} retries: #{e.message}"
+        end
       end
     end
   end

data/lib/eval_ruby/judges/openai.rb

@@ -9,30 +9,45 @@ module EvalRuby
     class OpenAI < Base
       API_URL = "https://api.openai.com/v1/chat/completions"
 
+      def initialize(config)
+        super
+        raise EvalRuby::Error, "API key is required. Set via EvalRuby.configure { |c| c.api_key = '...' }" if @config.api_key.nil? || @config.api_key.empty?
+      end
+
       def call(prompt)
-        uri = URI(API_URL)
-        request = Net::HTTP::Post.new(uri)
-        request["Authorization"] = "Bearer #{@config.api_key}"
-        request["Content-Type"] = "application/json"
-        request.body = JSON.generate({
-          model: @config.judge_model,
-          messages: [{role: "user", content: prompt}],
-          temperature: 0.0,
-          response_format: {type: "json_object"}
-        })
+        retries = 0
+        begin
+          uri = URI(API_URL)
+          request = Net::HTTP::Post.new(uri)
+          request["Authorization"] = "Bearer #{@config.api_key}"
+          request["Content-Type"] = "application/json"
+          request.body = JSON.generate({
+            model: @config.judge_model,
+            messages: [{role: "user", content: prompt}],
+            temperature: 0.0,
+            response_format: {type: "json_object"}
+          })
 
-        response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true,
-                                   read_timeout: @config.timeout) do |http|
-          http.request(request)
-        end
+          response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true,
+                                     read_timeout: @config.timeout) do |http|
+            http.request(request)
+          end
 
-        unless response.is_a?(Net::HTTPSuccess)
-          raise Error, "OpenAI API error: #{response.code} - #{response.body}"
-        end
+          unless response.is_a?(Net::HTTPSuccess)
+            raise Error, "OpenAI API error: #{response.code} - #{response.body}"
+          end
 
-        body = JSON.parse(response.body)
-        content = body.dig("choices", 0, "message", "content")
-        parse_json_response(content)
+          body = JSON.parse(response.body)
+          content = body.dig("choices", 0, "message", "content")
+          parse_json_response(content)
+        rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNRESET => e
+          retries += 1
+          if retries <= @config.max_retries
+            sleep(2 ** (retries - 1))
+            retry
+          end
+          raise EvalRuby::TimeoutError, "Judge API failed after #{@config.max_retries} retries: #{e.message}"
+        end
       end
     end
   end

data/lib/eval_ruby/metrics/context_precision.rb

@@ -20,7 +20,7 @@ module EvalRuby
       PROMPT
 
       def call(question:, context:, **_kwargs)
-        contexts = Array(context)
+        contexts = context.is_a?(Array) ? context : [context.to_s]
         return {score: 0.0, details: {}} if contexts.empty?
 
         contexts_text = contexts.each_with_index.map { |c, i| "[#{i}] #{c}" }.join("\n\n")

data/lib/eval_ruby/metrics/context_recall.rb

@@ -21,7 +21,7 @@ module EvalRuby
       PROMPT
 
       def call(context:, ground_truth:, **_kwargs)
-        contexts = Array(context)
+        contexts = context.is_a?(Array) ? context : [context.to_s]
         return {score: 0.0, details: {}} if contexts.empty?
 
         contexts_text = contexts.each_with_index.map { |c, i| "[#{i}] #{c}" }.join("\n\n")

data/lib/eval_ruby/metrics/faithfulness.rb

@@ -26,7 +26,7 @@ module EvalRuby
       PROMPT
 
       def call(answer:, context:, **_kwargs)
-        context_text = Array(context).join("\n\n")
+        context_text = context.is_a?(Array) ? context.join("\n\n") : context.to_s
         prompt = format(PROMPT_TEMPLATE, context: context_text, answer: answer)
 
         result = judge.call(prompt)

data/lib/eval_ruby/report.rb

@@ -32,7 +32,8 @@ module EvalRuby
         next if values.empty?
 
         mean = values.sum / values.size.to_f
-        variance = values.sum { |v| (v - mean)**2 } / values.size.to_f
+        denominator = values.size > 1 ? (values.size - 1).to_f : 1.0
+        variance = values.sum { |v| (v - mean)**2 } / denominator
         std = Math.sqrt(variance)
         hash[metric] = {mean: mean, std: std, min: values.min, max: values.max, count: values.size}
       end

data/lib/eval_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module EvalRuby
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/eval_ruby.rb

@@ -23,6 +23,9 @@ require_relative "eval_ruby/comparison"
 
 module EvalRuby
   class Error < StandardError; end
+  class APIError < Error; end
+  class TimeoutError < Error; end
+  class InvalidResponseError < Error; end
 
   class << self
     def configuration
@@ -79,16 +82,5 @@ module EvalRuby
     def compare(report_a, report_b)
       Comparison.new(report_a, report_b)
     end
-
-    private
-
-    def build_judge
-      config = configuration
-      case config.judge_llm
-      when :openai then Judges::OpenAI.new(config)
-      when :anthropic then Judges::Anthropic.new(config)
-      else raise Error, "Unknown judge LLM: #{config.judge_llm}"
-      end
-    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: eval-ruby
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Johannes Dwi Cahyo