evals 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a7eac12ccc87afa7859b543d5beaf530c30ef7fb935b92c0c79a4a0793963d01
-  data.tar.gz: db6974c59eac280dc28ac2c12b19ab06870aa9846bf5bb782c75b14f0792582b
+  metadata.gz: d605671c8627234dfb82834acb733e308b2cf9dc8f4ec1c9d3335d6cb63a42ab
+  data.tar.gz: fbff5f3dabd7f3721a09f068318d7912c642b75811673789bbee50bb1fbec31d
 SHA512:
-  metadata.gz: d889219abbcb887c15bc4265637f5b4dd14be1bcaa19f67daa475116dac0a66b2f149b35ac5396ef4ce94a71474eb360ea286c977700a9ba736398dc7d67d2fd
-  data.tar.gz: 867fefcb7103faa03d84c8240984040ab005fdbf1f551dff89dd41e82e248b32a1bd40cc3154dab949e33310d404c6731ec9f89f00d6dcf14d2b7c8634975d34
+  metadata.gz: d86068f9647b5a8f32dfc71f914f5e5afa74a98692c2d264a3c684ea9f9c096453767a5e0072adbe8e7d3d9440a35280e69de1e1165d22ef27054f6ad2ddc5d4
+  data.tar.gz: 5d4086f3a135e70d7e47f52b81ff9f411e0dc21bf47128ca0be9c14f417c98329f40ed004f90c7528951ce02cd6193ceb29b2ca16b2b6ea9c9c1ef54df6ed756

data/.env.example

@@ -0,0 +1 @@
+ANTHROPIC_API_KEY=

data/README.md

@@ -1,6 +1,8 @@
 # Evals
 
-A library for LLM evals
+A Ruby library for evaluating LLM responses.
+
+Based on the Prompt Evaluation example in the [Anthropic Skilljar course](https://anthropic.skilljar.com/claude-with-the-anthropic-api).
 
 ## Installation
 
@@ -16,9 +18,13 @@ If bundler is not being used to manage dependencies, install the gem by executin
 gem install evals
 ```
 
+## Configuration
+
+Copy `.env.example` to `.env` and add your Anthropic API key.
+
 ## Usage
 
-TODO: Write usage instructions here
+See [examples/demo.rb](examples/demo.rb) for an example.
 
 ## Development
 

data/examples/demo.rb

@@ -0,0 +1,130 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "json"
+require "dotenv/load"
+require "anthropic"
+require "evals"
+
+# Client Initialization and helper functions
+Dotenv.load
+
+CLIENT = Anthropic::Client.new(api_key: ENV["ANTHROPIC_API_KEY"])
+MODEL = "claude-3-5-haiku-latest"
+
+def add_user_message(messages, text)
+  messages << {role: "user", content: text}
+end
+
+def add_assistant_message(messages, text)
+  messages << {role: "assistant", content: text}
+end
+
+def chat(messages, system: nil, temperature: 1.0, stop_sequences: [])
+  params = {
+    model: MODEL,
+    max_tokens: 1000,
+    messages: messages,
+    temperature: temperature,
+    stop_sequences: stop_sequences
+  }
+
+  params[:system] = system if system
+
+  response = CLIENT.messages.create(params)
+  response.content[0].text
+end
+
+# Create an instance of Evals::PromptEvaluator
+# Increase `max_concurrent_tasks` for greater concurrency, but beware of rate limit errors!
+evaluator = Evals::PromptEvaluator.new(max_concurrent_tasks: 1)
+
+# Generate dataset
+evaluator.generate_dataset(
+  # Describe the purpose or goal of the prompt you're trying to test
+  "Write a compact, concise 1 day meal plan for a single athlete",
+  # Describe the different inputs that your prompt requires
+  prompt_inputs_spec: {
+    "height" => "Athlete's height in cm",
+    "weight" => "Athlete's weight in kg",
+    "goal" => "Goal of the athlete",
+    "restrictions" => "Dietary restrictions of the athlete"
+  },
+  # Where to write the generated dataset
+  output_file: "dataset.json",
+  # Number of test cases to generate (recommend keeping this low if you're getting rate limit errors)
+  num_cases: 1
+)
+
+# Define and run the prompt you want to evaluate, returning the raw model output
+# This function is executed once for each test case
+def run_prompt(prompt_inputs)
+  prompt = <<~PROMPT
+    Generate a one-day meal plan for an athlete that meets their dietary restrictions.
+
+    <athlete_information>
+    - Height: #{prompt_inputs["height"]}
+    - Weight: #{prompt_inputs["weight"]}
+    - Goal: #{prompt_inputs["goal"]}
+    - Dietary restrictions: #{prompt_inputs["restrictions"]}
+    </athlete_information>
+
+    Guidelines:
+    1. Include accurate daily calorie amount
+    2. Show protein, fat, and carb amounts
+    3. Specify when to eat each meal
+    4. Use only foods that fit restrictions
+    5. List all portion sizes in grams
+    6. Keep budget-friendly if mentioned
+
+    Here is an example with a sample input and an ideal output:
+    <sample_input>
+    height: 170
+    weight: 70
+    goal: Maintain fitness and improve cholesterol levels
+    restrictions: High cholesterol
+    </sample_input>
+    <ideal_output>
+    Here is a one-day meal plan for an athlete aiming to maintain fitness and improve cholesterol levels:
+
+    *   **Calorie Target:** Approximately 2500 calories
+    *   **Macronutrient Breakdown:** Protein (140g), Fat (70g), Carbs (340g)
+
+    **Meal Plan:**
+
+    *   **Breakfast (7:00 AM):** Oatmeal (80g dry weight) with berries (100g) and walnuts (15g). Skim milk (240g).
+        *   Protein: 15g, Fat: 15g, Carbs: 60g
+    *   **Mid-Morning Snack (10:00 AM):** Apple (150g) with almond butter (30g).
+        *   Protein: 7g, Fat: 18g, Carbs: 25g
+    *   **Lunch (1:00 PM):** Grilled chicken breast (120g) salad with mixed greens (150g), cucumber (50g), tomato (50g), and a light vinaigrette dressing (30g). Whole wheat bread (60g).
+        *   Protein: 40g, Fat: 15g, Carbs: 70g
+    *   **Afternoon Snack (4:00 PM):** Greek yogurt (170g, non-fat) with a banana (120g).
+        *   Protein: 20g, Fat: 0g, Carbs: 40g
+    *   **Dinner (7:00 PM):** Baked salmon (140g) with steamed broccoli (200g) and quinoa (75g dry weight).
+        *   Protein: 40g, Fat: 20g, Carbs: 80g
+    *   **Evening Snack (9:00 PM):** Small handful of almonds (20g).
+        *   Protein: 8g, Fat: 12g, Carbs: 15g
+
+    This meal plan prioritizes lean protein sources, whole grains, fruits, and vegetables, while limiting saturated and trans fats to support healthy cholesterol levels.
+    </ideal_output>
+    This example meal plan is well-structured, provides detailed information on food choices and quantities, and aligns with the athlete's goals and restrictions.
+  PROMPT
+
+  messages = []
+  add_user_message(messages, prompt)
+  chat(messages)
+end
+
+# Run evaluation
+evaluator.run_evaluation(
+  method(:run_prompt),
+  "dataset.json",
+  extra_criteria: <<~CRITERIA
+    The output should include:
+    - Daily caloric total
+    - Macronutrient breakdown
+    - Meals with exact foods, portions, and timing
+  CRITERIA
+)
+
+puts "Evaluation completed. Results saved to output.json and output.html"

data/lib/evals/prompt_evaluator.rb

@@ -0,0 +1,566 @@
+require "json"
+require "net/http"
+require "uri"
+
+module Evals
+  class PromptEvaluator
+    def initialize(max_concurrent_tasks: 3)
+      @max_concurrent_tasks = max_concurrent_tasks
+      @client = Anthropic::Client.new(api_key: ENV["ANTHROPIC_API_KEY"])
+      @model = "claude-3-5-haiku-latest"
+    end
+
+    def render(template_string, variables)
+      placeholders = template_string.scan(/{([^{}]+)}/)
+
+      result = template_string
+      placeholders.flatten.each do |placeholder|
+        if variables.key?(placeholder)
+          result = result.gsub("{#{placeholder}}", variables[placeholder].to_s)
+        end
+      end
+
+      result.gsub("{{", "{").gsub("}}", "}")
+    end
+
+    def add_user_message(messages, text)
+      messages << {role: "user", content: text}
+    end
+
+    def add_assistant_message(messages, text)
+      messages << {role: "assistant", content: text}
+    end
+
+    def chat(messages, system: nil, temperature: 1.0, stop_sequences: [])
+      params = {
+        model: @model,
+        max_tokens: 1000,
+        messages: messages,
+        temperature: temperature,
+        stop_sequences: stop_sequences
+      }
+
+      params[:system] = system if system
+
+      response = @client.messages.create(params)
+      response.content[0].text
+    end
+
+    def generate_unique_ideas(task_description, prompt_inputs_spec, num_cases)
+      prompt = <<~TEXT
+        Generate #{num_cases} unique, diverse ideas for testing a prompt that accomplishes this task:
+
+        <task_description>
+        #{task_description}
+        </task_description>
+
+        The prompt will receive the following inputs
+        <prompt_inputs>
+        #{prompt_inputs_spec}
+        </prompt_inputs>
+
+        Each idea should represent a distinct scenario or example that tests different aspects of the task.
+
+        Output Format:
+        Provide your response as a structured JSON array where each item is a brief description of the idea.
+
+        Example:
+        ```json
+        [
+            "Testing with technical computer science terminology",
+            "Testing with medical research findings",
+            "Testing with complex mathematical concepts",
+            ...
+        ]
+        ```
+
+        Ensure each idea is:
+        - Clearly distinct from the others
+        - Relevant to the task description
+        - Specific enough to guide generation of a full test case
+        - Quick to solve without requiring extensive computation or multi-step processing
+        - Solvable with no more than 400 tokens of output
+
+        Remember, only generate #{num_cases} unique ideas
+      TEXT
+
+      system_prompt = "You are a test scenario designer specialized in creating diverse, unique testing scenarios."
+
+      example_prompt_inputs = ""
+      prompt_inputs_spec.each do |key, value|
+        val = value.gsub("\n", "\\n")
+        example_prompt_inputs += "\"#{key}\": str # #{val},"
+      end
+
+      rendered_prompt = render(
+        prompt.strip,
+        {
+          "task_description" => task_description,
+          "num_cases" => num_cases,
+          "prompt_inputs" => example_prompt_inputs
+        }
+      )
+
+      messages = []
+      add_user_message(messages, rendered_prompt)
+      add_assistant_message(messages, "```json")
+      text = chat(
+        messages,
+        stop_sequences: ["```"],
+        system: system_prompt,
+        temperature: 1.0
+      )
+
+      JSON.parse(text)
+    end
+
+    def generate_test_case(task_description, idea, prompt_inputs_spec = {})
+      example_prompt_inputs = ""
+      prompt_inputs_spec.each do |key, value|
+        val = value.gsub("\n", "\\n")
+        example_prompt_inputs += "\"#{key}\": \"EXAMPLE_VALUE\", // #{val}\n"
+      end
+
+      allowed_keys = prompt_inputs_spec.keys.map { |key| "\"#{key}\"" }.join(", ")
+
+      prompt = <<~TEXT
+        Generate a single detailed test case for a prompt evaluation based on:
+
+        <task_description>
+        #{task_description}
+        </task_description>
+
+        <specific_idea>
+        #{idea}
+        </specific_idea>
+
+        <allowed_input_keys>
+        #{allowed_keys}
+        </allowed_input_keys>
+
+        Output Format:
+        ```json
+        {
+            "prompt_inputs": {
+            #{example_prompt_inputs}
+            },
+            "solution_criteria": ["criterion 1", "criterion 2", ...] // Concise list of criteria for evaluating the solution, 1 to 4 items
+        }
+        ```
+
+        IMPORTANT REQUIREMENTS:
+        - You MUST ONLY use these exact input keys in your prompt_inputs: #{allowed_keys}
+        - Do NOT add any additional keys to prompt_inputs
+        - All keys listed in allowed_input_keys must be included in your response
+        - Make the test case realistic and practically useful
+        - Include measurable, concise solution criteria
+        - The solution criteria should ONLY address the direct requirements of the task description and the generated prompt_inputs
+        - Avoid over-specifying criteria with requirements that go beyond the core task
+        - Keep solution criteria simple, focused, and directly tied to the fundamental task
+        - The test case should be tailored to the specific idea provided
+        - Quick to solve without requiring extensive computation or multi-step processing
+        - Solvable with no more than 400 tokens of output
+        - DO NOT include any fields beyond those specified in the output format
+      TEXT
+
+      system_prompt = "You are a test case creator specializing in designing evaluation scenarios."
+
+      rendered_prompt = render(
+        prompt.strip,
+        {
+          "allowed_keys" => allowed_keys,
+          "task_description" => task_description,
+          "idea" => idea,
+          "example_prompt_inputs" => example_prompt_inputs
+        }
+      )
+
+      messages = []
+      add_user_message(messages, rendered_prompt)
+      add_assistant_message(messages, "```json")
+      text = chat(
+        messages,
+        stop_sequences: ["```"],
+        system: system_prompt,
+        temperature: 0.7
+      )
+
+      test_case = JSON.parse(text)
+      test_case["task_description"] = task_description
+      test_case["scenario"] = idea
+
+      test_case
+    end
+
+    def generate_dataset(task_description, prompt_inputs_spec: {}, num_cases: 1, output_file: "dataset.json")
+      ideas = generate_unique_ideas(task_description, prompt_inputs_spec, num_cases)
+
+      dataset = []
+      completed = 0
+      total = ideas.length
+      last_reported_percentage = 0
+
+      threads = ideas.map do |idea|
+        Thread.new do
+          generate_test_case(task_description, idea, prompt_inputs_spec)
+        end
+      end
+
+      threads.each do |thread|
+        result = thread.value
+        completed += 1
+        current_percentage = ((completed.to_f / total) * 100).to_i
+        milestone_percentage = (current_percentage / 20) * 20
+
+        if milestone_percentage > last_reported_percentage
+          puts "Generated #{completed}/#{total} test cases"
+          last_reported_percentage = milestone_percentage
+        end
+
+        dataset << result
+      rescue => e
+        puts "Error generating test case: #{e}"
+      end
+
+      File.write(output_file, JSON.pretty_generate(dataset))
+      dataset
+    end
+
+    def grade_output(test_case, output, extra_criteria)
+      prompt_inputs = ""
+      test_case["prompt_inputs"].each do |key, value|
+        val = value.gsub("\n", "\\n")
+        prompt_inputs += "\"#{key}\":\"#{val}\",\n"
+      end
+
+      extra_criteria_section = ""
+      if extra_criteria
+        extra_criteria_template = <<~TEXT
+          Mandatory Requirements - ANY VIOLATION MEANS AUTOMATIC FAILURE (score of 3 or lower):
+          <extra_important_criteria>
+          #{extra_criteria}
+          </extra_important_criteria>
+        TEXT
+        extra_criteria_section = render(
+          extra_criteria_template.strip,
+          {"extra_criteria" => extra_criteria}
+        )
+      end
+
+      eval_template = <<~TEXT
+        Your task is to evaluate the following AI-generated solution with EXTREME RIGOR.
+
+        Original task description:
+        <task_description>
+        #{test_case["task_description"]}
+        </task_description>
+
+        Original task inputs:
+        <task_inputs>
+        { #{prompt_inputs} }
+        </task_inputs>
+
+        Solution to Evaluate:
+        <solution>
+        #{output}
+        </solution>
+
+        Criteria you should use to evaluate the solution:
+        <criteria>
+        #{test_case["solution_criteria"].join("\n")}
+        </criteria>
+
+        #{extra_criteria_section}
+
+        Scoring Guidelines:
+        * Score 1-3: Solution fails to meet one or more MANDATORY requirements
+        * Score 4-6: Solution meets all mandatory requirements but has significant deficiencies in secondary criteria
+        * Score 7-8: Solution meets all mandatory requirements and most secondary criteria, with minor issues
+        * Score 9-10: Solution meets all mandatory and secondary criteria
+
+        IMPORTANT SCORING INSTRUCTIONS:
+        * Grade the output based ONLY on the listed criteria. Do not add your own extra requirements.
+        * If a solution meets all of the mandatory and secondary criteria give it a 10
+        * Don't complain that the solution "only" meets the mandatory and secondary criteria. Solutions shouldn't go above and beyond - they should meet the exact listed criteria.
+        * ANY violation of a mandatory requirement MUST result in a score of 3 or lower
+        * The full 1-10 scale should be utilized - don't hesitate to give low scores when warranted
+
+        Output Format
+        Provide your evaluation as a structured JSON object with the following fields, in this specific order:
+        - "strengths": An array of 1-3 key strengths
+        - "weaknesses": An array of 1-3 key areas for improvement
+        - "reasoning": A concise explanation of your overall assessment
+        - "score": A number between 1-10
+
+        Respond with JSON. Keep your response concise and direct.
+      TEXT
+
+      eval_prompt = render(
+        eval_template.strip,
+        {
+          "task_description" => test_case["task_description"],
+          "prompt_inputs" => prompt_inputs,
+          "output" => output,
+          "solution_criteria" => test_case["solution_criteria"].join("\n"),
+          "extra_criteria_section" => extra_criteria_section
+        }
+      )
+
+      messages = []
+      add_user_message(messages, eval_prompt)
+      add_assistant_message(messages, "```json")
+      eval_text = chat(
+        messages,
+        stop_sequences: ["```"],
+        temperature: 0.0
+      )
+
+      JSON.parse(eval_text)
+    end
+
+    def run_test_case(test_case, run_prompt_function, extra_criteria = nil)
+      output = run_prompt_function.call(test_case["prompt_inputs"])
+
+      model_grade = grade_output(test_case, output, extra_criteria)
+      model_score = model_grade["score"]
+      reasoning = model_grade["reasoning"]
+
+      {
+        "output" => output,
+        "test_case" => test_case,
+        "score" => model_score,
+        "reasoning" => reasoning
+      }
+    end
+
+    def run_evaluation(run_prompt_function, dataset_file, extra_criteria: nil, json_output_file: "output.json", html_output_file: "output.html")
+      dataset = JSON.parse(File.read(dataset_file))
+
+      results = []
+      completed = 0
+      total = dataset.length
+      last_reported_percentage = 0
+
+      threads = dataset.map do |test_case|
+        Thread.new do
+          run_test_case(test_case, run_prompt_function, extra_criteria)
+        end
+      end
+
+      threads.each do |thread|
+        result = thread.value
+        completed += 1
+        current_percentage = ((completed.to_f / total) * 100).to_i
+        milestone_percentage = (current_percentage / 20) * 20
+
+        if milestone_percentage > last_reported_percentage
+          puts "Graded #{completed}/#{total} test cases"
+          last_reported_percentage = milestone_percentage
+        end
+
+        results << result
+      end
+
+      average_score = results.sum { |result| result["score"] } / results.length.to_f
+      puts "Average score: #{average_score}"
+
+      File.write(json_output_file, JSON.pretty_generate(results))
+
+      html = generate_prompt_evaluation_report(results)
+      File.write(html_output_file, html)
+
+      results
+    end
+
+    private
+
+    def generate_prompt_evaluation_report(evaluation_results)
+      total_tests = evaluation_results.length
+      scores = evaluation_results.map { |result| result["score"] }
+      avg_score = scores.empty? ? 0 : scores.sum.to_f / scores.length
+      max_possible_score = 10
+      pass_rate = if total_tests > 0
+        100.0 * scores.count { |s| s >= 7 } / total_tests
+      else
+        0
+      end
+
+      html = <<~HTML
+        <!DOCTYPE html>
+        <html lang="en">
+        <head>
+            <meta charset="UTF-8">
+            <meta name="viewport" content="width=device-width, initial-scale=1.0">
+            <title>Prompt Evaluation Report</title>
+            <style>
+                body {
+                    font-family: Arial, sans-serif;
+                    line-height: 1.6;
+                    margin: 0;
+                    padding: 20px;
+                    color: #333;
+                }
+                .header {
+                    background-color: #f0f0f0;
+                    padding: 20px;
+                    border-radius: 5px;
+                    margin-bottom: 20px;
+                }
+                .summary-stats {
+                    display: flex;
+                    justify-content: space-between;
+                    flex-wrap: wrap;
+                    gap: 10px;
+                }
+                .stat-box {
+                    background-color: #fff;
+                    border-radius: 5px;
+                    padding: 15px;
+                    box-shadow: 0 2px 5px rgba(0,0,0,0.1);
+                    flex-basis: 30%;
+                    min-width: 200px;
+                }
+                .stat-value {
+                    font-size: 24px;
+                    font-weight: bold;
+                    margin-top: 5px;
+                }
+                table {
+                    width: 100%;
+                    border-collapse: collapse;
+                    margin-top: 20px;
+                }
+                th {
+                    background-color: #4a4a4a;
+                    color: white;
+                    text-align: left;
+                    padding: 12px;
+                }
+                td {
+                    padding: 10px;
+                    border-bottom: 1px solid #ddd;
+                    vertical-align: top;
+                }
+                tr:nth-child(even) {
+                    background-color: #f9f9f9;
+                }
+                .output-cell {
+                    white-space: pre-wrap;
+                }
+                .score {
+                    font-weight: bold;
+                    padding: 5px 10px;
+                    border-radius: 3px;
+                    display: inline-block;
+                }
+                .score-high {
+                    background-color: #c8e6c9;
+                    color: #2e7d32;
+                }
+                .score-medium {
+                    background-color: #fff9c4;
+                    color: #f57f17;
+                }
+                .score-low {
+                    background-color: #ffcdd2;
+                    color: #c62828;
+                }
+                .output {
+                    overflow: auto;
+                    white-space: pre-wrap;
+                }
+                .output pre {
+                    background-color: #f5f5f5;
+                    border: 1px solid #ddd;
+                    border-radius: 4px;
+                    padding: 10px;
+                    margin: 0;
+                    font-family: 'Consolas', 'Monaco', 'Courier New', monospace;
+                    font-size: 14px;
+                    line-height: 1.4;
+                    color: #333;
+                    box-shadow: inset 0 1px 3px rgba(0, 0, 0, 0.1);
+                    overflow-x: auto;
+                    white-space: pre-wrap;
+                    word-wrap: break-word;
+                }
+                td {
+                    width: 20%;
+                }
+                .score-col {
+                    width: 80px;
+                }
+            </style>
+        </head>
+        <body>
+            <div class="header">
+                <h1>Prompt Evaluation Report</h1>
+                <div class="summary-stats">
+                    <div class="stat-box">
+                        <div>Total Test Cases</div>
+                        <div class="stat-value">#{total_tests}</div>
+                    </div>
+                    <div class="stat-box">
+                        <div>Average Score</div>
+                        <div class="stat-value">#{sprintf("%.1f", avg_score)} / #{max_possible_score}</div>
+                    </div>
+                    <div class="stat-box">
+                        <div>Pass Rate (≥7)</div>
+                        <div class="stat-value">#{sprintf("%.1f", pass_rate)}%</div>
+                    </div>
+                </div>
+            </div>
+
+            <table>
+                <thead>
+                    <tr>
+                        <th>Scenario</th>
+                        <th>Prompt Inputs</th>
+                        <th>Solution Criteria</th>
+                        <th>Output</th>
+                        <th>Score</th>
+                        <th>Reasoning</th>
+                    </tr>
+                </thead>
+                <tbody>
+      HTML
+
+      evaluation_results.each do |result|
+        prompt_inputs_html = result["test_case"]["prompt_inputs"].map do |key, value|
+          "<strong>#{key}:</strong> #{value}"
+        end.join("<br>")
+
+        criteria_string = result["test_case"]["solution_criteria"].join("<br>• ")
+
+        score = result["score"]
+        score_class = if score >= 8
+          "score-high"
+        elsif score <= 5
+          "score-low"
+        else
+          "score-medium"
+        end
+
+        html += <<~HTML
+          <tr>
+              <td>#{result["test_case"]["scenario"]}</td>
+              <td class="prompt-inputs">#{prompt_inputs_html}</td>
+              <td class="criteria">• #{criteria_string}</td>
+              <td class="output"><pre>#{result["output"]}</pre></td>
+              <td class="score-col"><span class="score #{score_class}">#{score}</span></td>
+              <td class="reasoning">#{result["reasoning"]}</td>
+          </tr>
+        HTML
+      end
+
+      html += <<~HTML
+                </tbody>
+            </table>
+        </body>
+        </html>
+      HTML
+
+      html
+    end
+  end
+end

data/lib/evals/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Evals
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/evals.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "evals/version"
+require_relative "evals/prompt_evaluator"
 
 module Evals
   class Error < StandardError; end

metadata

@@ -1,26 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: evals
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Andy Waite
 bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
-dependencies: []
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: anthropic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 email:
 - 13400+andyw8@users.noreply.github.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".env.example"
 - ".ruby-version"
 - ".standard.yml"
 - LICENSE.txt
 - README.md
 - Rakefile
+- examples/demo.rb
 - lib/evals.rb
+- lib/evals/prompt_evaluator.rb
 - lib/evals/version.rb
 - sig/evals.rbs
 homepage: https://github.com/andyw8/evals