ruby_llm-agent-evaluations 0.2.0

data/db/migrate/20261011000000_create_agent_evaluation_history.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+class CreateAgentEvaluationHistory < ActiveRecord::Migration[7.1]
+  RUNS = "ruby_llm_evaluations_runs"
+  CASES = "ruby_llm_evaluations_case_results"
+  METRICS = "ruby_llm_evaluations_metric_results"
+  LEGACY = "ruby_llm_evaluations_legacy_runs"
+  LEGACY_VERSIONS = "ruby_llm_evaluations_schema_versions"
+
+  class LegacyRun < ActiveRecord::Base
+    self.table_name = LEGACY
+    self.primary_key = "sequence"
+  end
+
+  class LegacyVersion < ActiveRecord::Base
+    self.table_name = LEGACY_VERSIONS
+    self.primary_key = "version"
+  end
+
+  class MigratedRun < ActiveRecord::Base
+    self.table_name = RUNS
+  end
+
+  class MigratedCase < ActiveRecord::Base
+    self.table_name = CASES
+  end
+
+  class MigratedMetric < ActiveRecord::Base
+    self.table_name = METRICS
+  end
+
+  def up
+    raise ActiveRecord::MigrationError, "evaluation history requires PostgreSQL" unless connection.adapter_name == "PostgreSQL"
+
+    legacy = table_exists?(RUNS) && column_exists?(RUNS, :payload)
+    if legacy
+      unless table_exists?(LEGACY_VERSIONS) && LegacyVersion.pluck(:version) == [1]
+        raise ActiveRecord::MigrationError, "unsupported legacy evaluation history version"
+      end
+      rename_table RUNS, LEGACY
+    elsif table_exists?(RUNS)
+      raise ActiveRecord::MigrationError, "evaluation model tables already exist without this migration"
+    end
+
+    create_table RUNS, id: :string do |table|
+      table.bigserial :sequence, null: false
+      table.string :suite, null: false
+      table.string :agent, null: false
+      table.string :version, null: false, default: "1"
+      table.string :suite_digest, null: false
+      table.column :started_at, :timestamptz, null: false, precision: 6
+      table.column :finished_at, :timestamptz, null: false, precision: 6
+      table.string :status, null: false
+      table.jsonb :metadata, null: false, default: {}
+      table.jsonb :summary, null: false
+      table.timestamps precision: 6
+    end
+    add_index RUNS, :sequence, unique: true
+    add_index RUNS, [:started_at, :sequence], order: { started_at: :desc, sequence: :desc }, name: "rle_runs_recent"
+    add_index RUNS, [:suite, :agent, :suite_digest, :started_at, :sequence],
+              order: { started_at: :desc, sequence: :desc }, name: "rle_runs_comparable"
+
+    create_table CASES do |table|
+      table.references :run, type: :string, null: false, foreign_key: { to_table: RUNS }
+      table.integer :position, null: false
+      table.string :case_id, null: false
+      table.text :name, null: false
+      table.jsonb :question
+      table.jsonb :expected_answer
+      table.jsonb :expected_actions
+      table.jsonb :criteria, null: false, default: {}
+      table.jsonb :metadata, null: false, default: {}
+      table.jsonb :output
+      table.string :status, null: false
+      table.float :duration_ms, null: false
+      table.bigint :input_tokens
+      table.bigint :output_tokens
+      table.float :cost
+      table.jsonb :actions
+      table.jsonb :response_metadata
+      table.jsonb :error_details
+      table.timestamps precision: 6
+    end
+    add_index CASES, [:run_id, :position], unique: true, name: "rle_case_order"
+    add_index CASES, [:run_id, :case_id], unique: true, name: "rle_case_identity"
+
+    create_table METRICS do |table|
+      table.references :case_result, null: false, foreign_key: { to_table: CASES }
+      table.integer :position, null: false
+      table.string :name, null: false
+      table.float :threshold, null: false
+      table.float :score
+      table.boolean :passed, null: false
+      table.text :reason
+      table.jsonb :error_details
+      table.timestamps precision: 6
+    end
+    add_index METRICS, [:case_result_id, :position], unique: true, name: "rle_metric_order"
+    add_index METRICS, [:case_result_id, :name], unique: true, name: "rle_metric_identity"
+
+    if legacy
+      copy_legacy_history
+      connection.reset_pk_sequence!(RUNS, "sequence", "#{RUNS}_sequence_seq")
+      drop_table LEGACY
+    end
+    drop_table LEGACY_VERSIONS if table_exists?(LEGACY_VERSIONS)
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration, "evaluation history migration preserves append-only evidence"
+  end
+
+  private
+
+  def copy_legacy_history
+    [LegacyRun, MigratedRun, MigratedCase, MigratedMetric].each(&:reset_column_information)
+    LegacyRun.find_each do |row|
+      value = row.payload
+      run = MigratedRun.create!(
+        id: value.fetch("id"), sequence: row.sequence, suite: value.fetch("suite"), agent: value.fetch("agent"),
+        version: value.fetch("version", "1"), suite_digest: value.fetch("suite_digest"),
+        started_at: value.fetch("started_at"), finished_at: value.fetch("finished_at"),
+        status: value.fetch("status"), metadata: value.fetch("metadata", {}), summary: value.fetch("summary")
+      )
+      value.fetch("cases").each_with_index do |item, position|
+        metrics = item.fetch("evaluations")
+        question = MigratedCase.create!(
+          run_id: run.id, position: position, case_id: item.fetch("id"), name: item.fetch("name"),
+          question: item["input"], expected_answer: item["expected"], expected_actions: item["expected_actions"],
+          criteria: metrics.to_h { |metric| [metric.fetch("name"), metric.fetch("threshold")] },
+          metadata: item.fetch("metadata", {}), output: item["output"], status: item.fetch("status"),
+          duration_ms: item.fetch("duration_ms"), input_tokens: item["input_tokens"], output_tokens: item["output_tokens"],
+          cost: item["cost"], actions: item["actions"], response_metadata: item["response_metadata"], error_details: item["error"]
+        )
+        metrics.each_with_index do |metric, metric_position|
+          MigratedMetric.create!(
+            case_result_id: question.id, position: metric_position, name: metric.fetch("name"),
+            threshold: metric.fetch("threshold"), score: metric["score"], passed: metric.fetch("passed"),
+            reason: metric["reason"], error_details: metric["error"]
+          )
+        end
+      end
+    end
+  end
+end

data/db/migrate/20261011000100_rename_case_criteria_to_minimum_scores.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class RenameCaseCriteriaToMinimumScores < ActiveRecord::Migration[7.1]
+  def change
+    rename_column :ruby_llm_evaluations_case_results, :criteria, :minimum_scores
+  end
+end

data/examples/api_replay_eval.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "ruby_llm/evaluations"
+
+# Real HTTP request construction and parsing; the framework supplies the response.
+agent = lambda do |question|
+  order_id = ENV["AGENT_REVISION"] == "regressed" ? "99" : question.fetch(:order_id)
+  uri = URI("https://shipping.example/orders/#{order_id}")
+  uri.query = URI.encode_www_form(include_events: "true")
+  response = Net::HTTP.get_response(uri)
+  data = JSON.parse(response.body)
+  RubyLLM::Evaluations::Response.new(
+    output: data.fetch("status"),
+    actions: [{ group: "shipping", name: "lookup_order", arguments: { order_id: order_id } }]
+  )
+end
+
+RubyLLM::Evaluations.define("Shipping API replay", agent: agent, agent_name: "ShippingAgent") do
+  evaluate :exact_match, using: RubyLLM::Evaluations::Evaluators.exact_match
+  question({ order_id: "42" }, name: "File-backed order", expected_answer: "in_transit",
+           minimum_scores: { exact_match: 1.0, action_group: 1.0 },
+           expected_actions: [{ group: "shipping", name: "lookup_order", arguments: { order_id: "42" } }]) do
+    api_interactions do
+      expect_request(method: :get, url: "https://shipping.example/orders/42",
+                     query: { include_events: "true" }).respond_with(
+        body_file: "fixtures/order-42.json"
+      )
+    end
+  end
+
+  question({ order_id: "43" }, name: "Inline order", expected_answer: "delivered") do
+    api_interactions do
+      expect_request(method: :get, url: "https://shipping.example/orders/43",
+                     query: { include_events: "true" }).respond_with(
+        json: { id: "43", status: "delivered" }
+      )
+    end
+  end
+end

data/examples/context_bloat_eval.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "ruby_llm"
+require "ruby_llm/evaluations"
+require "net/http"
+require "json"
+
+module ContextBloatExample
+  E = RubyLLM::Evaluations
+  MAX_TOOL_RESULT_BYTES = 4_096
+
+  class ReturnPolicy < RubyLLM::Tool
+    description "Read the current product return deadline"
+    param :product, desc: "Product name"
+    attr_reader :api_response_bytes
+
+    def initialize(raw: false)
+      super()
+      @raw = raw
+    end
+
+    def name = "return_policy"
+
+    def execute(product:)
+      uri = URI("https://policy.example/returns")
+      uri.query = URI.encode_www_form(product: product)
+      body = Net::HTTP.get_response(uri).body
+      @api_response_bytes = body.bytesize
+      return body if @raw
+
+      data = JSON.parse(body)
+      # Preserve the relevant facts at the end, rather than truncating the beginning.
+      JSON.generate(product: data.fetch("product"), return_days: data.fetch("return_days"))
+    end
+  end
+
+  def self.suite(context:, model:, allow_host:, raw: false, expected_answer: "30")
+    agent = lambda do |question|
+      tool = ReturnPolicy.new(raw: raw)
+      chat = context.chat(model: model, provider: :openai, assume_model_exists: true)
+        .with_instructions("Use return_policy for the book. Reply with only the numeric return deadline.")
+        .with_tools(tool)
+      adapter = E::Adapters.ruby_llm_chat(action_groups: { return_policy: "returns" }) { chat }
+      response = adapter.call(question)
+      tool_bytes = chat.messages.select { |message| message.role == :tool }.map { |message| message.content.to_s.bytesize }
+      E::Response.new(output: response.output, actions: response.actions,
+                     input_tokens: response.input_tokens, output_tokens: response.output_tokens, cost: response.cost,
+                     metadata: response.metadata.merge("tool_result_bytes" => tool_bytes,
+                                                       "api_response_bytes" => tool.api_response_bytes))
+    end
+
+    E.define("Large tool result context budget", agent: agent, agent_name: "ReturnPolicyAgent") do
+      evaluate :exact_match, using: E::Evaluators.exact_match
+      evaluate :tool_result_budget do |_output, context|
+        sizes = context.response.metadata.fetch("tool_result_bytes")
+        raise E::Error, "tool-result instrumentation missing" if sizes.empty?
+        sizes.all? { |bytes| bytes <= context.metadata.fetch(:max_tool_result_bytes) }
+      end
+      question "What is the return deadline for my book?", expected_answer: expected_answer,
+               minimum_scores: { exact_match: 1.0, action_group: 1.0, tool_result_budget: 1.0 },
+               metadata: { max_tool_result_bytes: MAX_TOOL_RESULT_BYTES },
+               expected_actions: [{ group: "returns", name: "return_policy", arguments: { product: "book" } }] do
+        api_interactions(allow_hosts: [allow_host]) do
+          expect_request(method: :get, url: "https://policy.example/returns", query: { product: "book" })
+            .respond_with(body_file: "fixtures/large_return_policy.json")
+        end
+      end
+    end
+  end
+end
+
+# Loading this file for a local transport test does not configure or call a provider.
+if ENV["RUN_CONTEXT_BLOAT_EVAL"] == "1"
+  context = RubyLLM.context { |config| config.openai_api_key = ENV.fetch("OPENAI_API_KEY") }
+  ContextBloatExample.suite(context: context, model: ENV.fetch("OPENAI_MODEL", "gpt-4.1-mini"),
+                           allow_host: "api.openai.com", raw: ENV["AGENT_REVISION"] == "regressed")
+end