ask-eval 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12e438efcef2038b328af35cdb14d6741a7730940f1d8f7014f4550c3b8ac543
-  data.tar.gz: c32aaf27235fdd33e206ebc4434ed647015c8fa6d81a8b16b88546c36597f4d2
+  metadata.gz: d396e80eace74425011173335621532f318eb2921600aa8cfcdf1e79480f4df6
+  data.tar.gz: afc9c5a9873ba2b97766cd873211d4e2e94334c9adc8a39b372f4981a84e0fb1
 SHA512:
-  metadata.gz: 015cd82d67ac7fcf88c1661d352c459845ffbee6dc373e2d9999db68d8256cb1e63988618e47263ee59eb887fa7381e42b30b0b3fcc4c1fac4cdc802891f08b2
-  data.tar.gz: 8daf2d3ccc31244b1e3748ee2f1b07970a9c1af12592e9e6bddfdafaea980816961f4099f4382f5704bb47554e8036185ce4131fe4494400b9ba0b59bec4b026
+  metadata.gz: 88ca899713aecf0c9eec29f6470a7b0856e648f9cf54337d9252364c4ae01cd3df8f1040f5e534db7f2635e3ab8de365957858b01a11fd81d021acb81b82dde3
+  data.tar.gz: e05a44bd73f3f0236349cead4fa6f87bfe035a772f755b9ea379c3f165b01e78bb25f0cc28acbea443dc33d458031b62586bdeb61af7b35f14c897fa147998f6

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+## [0.4.0] — 2026-08-07
+
+### Added
+
+- **Datasets & experiment runs — the A/B-testing loop for agents.**
+  - `Ask::Eval::Dataset` — a pinned set of fixed inputs (like fixtures, but
+    the items are prompts/tasks fed to a live agent). `add` items with
+    optional `expected` output, `context`, `tags`, and `metadata`;
+    `Dataset.save` / `Dataset.load` persist to JSON files.
+  - `Ask::Eval::Experiment` — runs a dataset against a single variant: a
+    `runner:` callable turns each input into the agent's output, an
+    optional `scorer:` callable scores it (0..1), and per-item results
+    record output, score, error, and duration. Runner errors are captured
+    per item and the run continues.
+  - `Experiment#summary` (total/passed/failed/avg_score/duration) and
+    `Experiment#compare(other)` — side-by-side per-item deltas plus an
+    aggregate verdict ("a" / "b" / "tie"), so changing a prompt or model
+    and re-running the same dataset shows the improvement.
+
 ## [0.3.0] — 2026-08-03
 
 ### Added

data/lib/ask/eval/dataset.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Ask
+  module Eval
+    # A pinned set of fixed inputs for repeatable agent evaluation.
+    #
+    # Like Rails fixtures, a dataset gives you a deterministic baseline —
+    # but the items are *questions* (prompts/tasks fed to a live agent),
+    # not database records, and the point is running the same items against
+    # multiple variants (a changed prompt, a different model) and comparing
+    # the results.
+    #
+    # @example
+    #   dataset = Ask::Eval::Dataset.new("support-cases")
+    #   dataset.add(input: "The API returns 401 on stale tokens — how do I fix my auth flow?", tags: ["auth"])
+    #   dataset.add(input: "Summarize this thread and suggest next steps", expected: "A summary plus actions")
+    #   dataset.save("support-cases.json")
+    #
+    #   reloaded = Ask::Eval::Dataset.load("support-cases.json")
+    class Dataset
+      # A single dataset item: the input fed to the agent, optional expected
+      # output, optional context, and tags/metadata for filtering.
+      Item = Data.define(:id, :input, :expected, :context, :tags, :metadata) do
+        def to_h = { id: id, input: input, expected: expected, context: context, tags: tags, metadata: metadata }
+      end
+
+      # @param name [String] dataset name (used in reports)
+      # @param items [Array<Hash>] initial items with symbol or string keys
+      #   ({input:, expected:, context:, tags:, metadata:})
+      def initialize(name, items: [])
+        @name = name.to_s
+        @items = []
+        items.each { |item| add(**symbolize(item)) }
+      end
+
+      # @return [String] the dataset name
+      attr_reader :name
+
+      # Add an item.
+      #
+      # @param input [String] the prompt/task fed to the agent (required)
+      # @param expected [String, nil] expected/reference output
+      # @param context [String, nil] source context
+      # @param tags [Array<String>] labels for filtering
+      # @param metadata [Hash] free-form
+      # @return [Item]
+      # @raise [ArgumentError] on empty input
+      def add(input:, id: nil, expected: nil, context: nil, tags: [], metadata: {})
+        raise ArgumentError, "input is required" if input.to_s.strip.empty?
+
+        item = Item.new(
+          id: id || "item_#{@items.size + 1}",
+          input: input.to_s,
+          expected: expected,
+          context: context,
+          tags: Array(tags),
+          metadata: metadata
+        )
+        @items << item
+        item
+      end
+
+      # @return [Array<Item>] all items, in order
+      def items = @items.dup
+
+      # @return [Integer] number of items
+      def size = @items.size
+
+      def each(&block) = @items.each(&block)
+
+      # Create an experiment over this dataset (see {Experiment}).
+      #
+      # @param runner [Proc] called with the item's input; returns the
+      #   agent's output string
+      # @param scorer [Proc, nil] called with (input:, output:, expected:);
+      #   returns a score (0..1)
+      # @return [Ask::Eval::Experiment]
+      def experiment(runner:, scorer: nil)
+        Experiment.new(self, runner: runner, scorer: scorer)
+      end
+
+      # @return [Hash] serialized form (for {Dataset.save})
+      def to_h
+        { name: @name, items: @items.map(&:to_h) }
+      end
+
+      # Write the dataset to a JSON file.
+      #
+      # @param path [String]
+      # @return [void]
+      def save(path)
+        File.write(path, JSON.pretty_generate(to_h))
+        nil
+      end
+
+      # Load a dataset from a JSON file written by {Dataset.save} (or any
+      # compatible {name:, items:} shape).
+      #
+      # @param path [String]
+      # @return [Ask::Eval::Dataset]
+      def self.load(path)
+        data = JSON.parse(File.read(path))
+        new(data["name"] || File.basename(path, ".json"), items: data["items"] || [])
+      end
+
+      private
+
+      def symbolize(obj)
+        case obj
+        when Hash
+          obj.each_with_object({}) { |(k, v), h| h[k.to_sym] = symbolize(v) }
+        when Array
+          obj.map { |e| symbolize(e) }
+        else
+          obj
+        end
+      end
+    end
+  end
+end

data/lib/ask/eval/experiment.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module Ask
+  module Eval
+    # Runs a {Dataset} against a single variant and compares runs.
+    #
+    # An experiment executes every dataset item through a +runner+ (a
+    # callable that turns an input into the agent's output), optionally
+    # scores each output, and collects per-item results. Running the same
+    # dataset with two different runners (a changed prompt, a different
+    # model) and comparing the experiments is the A/B-testing loop of
+    # eval-driven development.
+    #
+    # @example
+    #   run_a = dataset.experiment(runner: ->(input) { agent(input, prompt: old) },
+    #                              scorer: judge_scorer)
+    #   run_b = dataset.experiment(runner: ->(input) { agent(input, prompt: new) },
+    #                              scorer: judge_scorer)
+    #   run_a.run
+    #   run_a.summary          # => {total:, passed:, failed:, avg_score:, ...}
+    #   run_a.compare(run_b)   # => per-item side-by-side + aggregate verdict
+    class Experiment
+      # One item's outcome: the produced output, optional score (0..1),
+      # error (when the runner raised), and wall-clock duration.
+      Result = Data.define(:item, :output, :score, :error, :duration_ms) do
+        # @return [Boolean] no error and, when scored, at least 0.5
+        def passed?
+          error.nil? && (score.nil? || score >= 0.5)
+        end
+      end
+
+      # @param dataset [Ask::Eval::Dataset]
+      # @param runner [Proc] called with the item's input; returns the
+      #   agent's output string
+      # @param scorer [Proc, nil] called with (input:, output:, expected:);
+      #   returns a score (0..1) or nil to skip scoring
+      def initialize(dataset, runner:, scorer: nil)
+        @dataset = dataset
+        @runner = runner
+        @scorer = scorer
+        @results = nil
+      end
+
+      # @return [Ask::Eval::Dataset]
+      attr_reader :dataset
+
+      # Execute every dataset item through the runner. A runner error is
+      # recorded on that item and the run continues.
+      #
+      # @return [self]
+      def run
+        @results = @dataset.items.map do |item|
+          start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          output = nil
+          error = nil
+          begin
+            output = @runner.call(item.input).to_s
+          rescue StandardError => e
+            error = e.message
+          end
+
+          score = nil
+          if error.nil? && @scorer
+            score = @scorer.call(input: item.input, output: output, expected: item.expected)
+          end
+
+          duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000).round
+          Result.new(item: item, output: output, score: score, error: error, duration_ms: duration_ms)
+        end
+        self
+      end
+
+      # @return [Array<Result>] per-item results (runs first if needed)
+      def results
+        @results || run.results
+      end
+
+      # @return [Hash] aggregate statistics
+      def summary
+        rs = results
+        scores = rs.filter_map(&:score)
+        {
+          total: rs.size,
+          passed: rs.count(&:passed?),
+          failed: rs.count { |r| !r.passed? },
+          avg_score: scores.empty? ? nil : (scores.sum.fdiv(scores.size)).round(3),
+          total_duration_ms: rs.sum(&:duration_ms)
+        }
+      end
+
+      # Side-by-side comparison with another experiment over the same (or
+      # overlapping) dataset items.
+      #
+      # @param other [Ask::Eval::Experiment]
+      # @return [Hash] {deltas: [{item_id:, input:, a:, b:, delta:}],
+      #   a: summary, b: summary, verdict: "a"|"b"|"tie"}
+      def compare(other)
+        a = results
+        b = other.results
+        ha = a.to_h { |r| [r.item.id, r] }
+        hb = b.to_h { |r| [r.item.id, r] }
+
+        deltas = (ha.keys & hb.keys).map do |id|
+          ra = ha[id]
+          rb = hb[id]
+          {
+            item_id: id,
+            input: ra.item.input,
+            a: { output: ra.output, score: ra.score, error: ra.error },
+            b: { output: rb.output, score: rb.score, error: rb.error },
+            delta: (ra.score && rb.score) ? (rb.score - ra.score).round(3) : nil
+          }
+        end
+
+        { deltas: deltas, a: summary, b: other.summary, verdict: verdict(summary, other.summary) }
+      end
+
+      private
+
+      def verdict(a, b)
+        return nil if a[:avg_score].nil? || b[:avg_score].nil?
+        return "a" if a[:avg_score] > b[:avg_score]
+        return "b" if b[:avg_score] > a[:avg_score]
+
+        "tie"
+      end
+    end
+  end
+end

data/lib/ask/eval/recorder.rb

@@ -2,6 +2,7 @@
 
 require "json"
 require "fileutils"
+require "time" # Time#iso8601 for recorded_at timestamps
 
 module Ask
   module Eval

data/lib/ask/eval/version.rb

@@ -1,5 +1,5 @@
 module Ask
   module Eval
-    VERSION = "0.3.0"
+    VERSION = "0.4.0"
   end
 end

data/lib/ask/eval.rb

@@ -14,6 +14,8 @@ module Ask
     autoload :Configuration, "ask/eval/configuration"
     autoload :Recorder, "ask/eval/recorder"
     autoload :SessionEval, "ask/eval/session_eval"
+    autoload :Dataset, "ask/eval/dataset"
+    autoload :Experiment, "ask/eval/experiment"
 
     # These are loaded eagerly since they define sub-modules with autoloads
     require_relative "eval/assertions"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-eval
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Kaka Ruto
@@ -51,6 +51,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: ask-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.1'
 description: 'Test LLM outputs with Minitest-native assertions. LLM-as-judge for faithfulness,
   hallucination, bias, toxicity. Deterministic assertions (contains, regex, JSON).
   CI-native: GitHub annotations, JUnit output, cost tracking.'
@@ -70,7 +84,9 @@ files:
 - lib/ask/eval/assertions/judge.rb
 - lib/ask/eval/configuration.rb
 - lib/ask/eval/cost_tracker.rb
+- lib/ask/eval/dataset.rb
 - lib/ask/eval/dsl.rb
+- lib/ask/eval/experiment.rb
 - lib/ask/eval/judge.rb
 - lib/ask/eval/judges/bias.rb
 - lib/ask/eval/judges/correctness.rb