llm-evals 0.4.0

data/lib/llm/evals/templates/html_report.html.erb

@@ -0,0 +1,184 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <title><%= escape(report_title) %></title>
+    <%= stylesheet_tag %>
+  </head>
+  <body>
+    <main>
+      <header class="report-header" id="overview">
+        <div>
+          <p class="eyebrow">Evaluation report</p>
+          <h1><%= escape(report_title) %></h1>
+          <p class="muted"><%= escape(configuration_name) %> · generated <%= Time.now.strftime("%Y-%m-%d %H:%M %Z") %></p>
+        </div>
+        <a class="top-link" href="#overview" aria-label="Back to report overview">↑ Top</a>
+      </header>
+
+      <nav class="section-nav" aria-label="Report sections">
+        <a href="#overview">Overview</a>
+        <a href="#standings">Standings</a>
+        <a href="#coverage">Coverage</a>
+        <% datasets.each_with_index do |dataset, index| %>
+          <a href="#dataset-<%= index + 1 %>"><%= escape(dataset[:name]) %></a>
+        <% end %>
+      </nav>
+
+      <section class="health" aria-label="Run health">
+        <div class="stat"><b><%= health[:successful] %>/<%= health[:total] %></b><span>model calls scored</span></div>
+        <div class="stat"><b><%= health[:failed] %></b><span>failed calls</span></div>
+        <div class="stat"><b>↓ <%= escape(health[:input_tokens]) %></b><span>tokens in</span></div>
+        <div class="stat"><b>↑ <%= escape(health[:output_tokens]) %></b><span>tokens out</span></div>
+        <div class="stat"><b><%= health[:cost] %></b><span>recorded cost</span></div>
+      </section>
+      <% if health[:failed].positive? %>
+        <p class="notice">This run has failed or unscored cases. A global leader requires full, error-free coverage.</p>
+      <% end %>
+
+      <section id="standings">
+        <h2>Global standings across all cases in this run</h2>
+        <p class="muted"><%= escape(weight_description) %></p>
+
+        <% if standings[:leader] %>
+          <% leader = standings[:leader] %>
+          <% leader_provider, leader_model = model_parts(leader[:model]) %>
+          <div class="leader">
+            <span class="eyebrow">ALL-CASE LEADER</span>
+            <strong><span class="provider"><%= leader_provider %></span>/<span class="model-id"><%= leader_model %></span></strong>
+            <span><%= percent(ranking_value(leader)) %> · Effort <%= effort_label(leader[:model]) %> · Temperature <%= temperature_label(leader[:model]) %> · <%= leader[:successful] %>/<%= standings[:expected_count] %> cases · <%= money(leader[:cost]) %></span>
+          </div>
+        <% else %>
+          <% best_coverage = standings[:best_coverage] %>
+          <div class="leader none">
+            <span class="eyebrow">ALL-CASE LEADER</span>
+            <strong>No leader established</strong>
+            <% if best_coverage %>
+              <% best_provider, best_model = model_parts(best_coverage[:model]) %>
+              <span>No contender covered all <%= standings[:expected_count] %> cases without failures. Best coverage: <%= best_coverage[:successful] %>/<%= standings[:expected_count] %> from <span class="provider"><%= best_provider %></span>/<span class="model-id"><%= best_model %></span> (effort <%= effort_label(best_coverage[:model]) %>, temperature <%= temperature_label(best_coverage[:model]) %>).</span>
+            <% else %>
+              <span>No model results are available for ranking.</span>
+            <% end %>
+          </div>
+        <% end %>
+
+        <% if standings[:ranked].empty? %>
+          <p class="muted">No fully covered contenders to plot.</p>
+        <% else %>
+          <div class="scroll">
+            <table class="sortable standings-table" aria-label="Model standings, initially ordered highest score first">
+              <thead>
+                <tr>
+                  <th><span>Provider</span><span>(vendor)</span></th>
+                  <th><span>Model</span><span>(ID and settings)</span></th>
+                  <th><span>Weighted</span><span>score</span></th>
+                  <th><span>Primary score</span><span>(<%= escape(primary_metric) %>)</span></th>
+                  <th title="Input tokens"><span>↓ Tokens in</span><span>(input)</span></th>
+                  <th title="Output tokens"><span>↑ Tokens out</span><span>(output)</span></th>
+                  <th class="standings-bar"><span>Score</span><span>bar</span></th>
+                </tr>
+              </thead>
+              <tbody>
+                <% standings[:ranked].reverse_each do |entry| %>
+                  <% provider, model = model_parts(entry[:model]) %>
+                  <% ranking = ranking_value(entry).to_f %>
+                  <% weighted_label = weighted_score ? percent(ranking_value(entry)) : "—" %>
+                  <tr>
+                    <td><span class="provider"><%= provider %></span></td><td><span class="model-id"><%= model %></span> <span class="standings-settings">(<%= effort_label(entry[:model]) %>, t=<%= temperature_label(entry[:model]) %>)</span></td>
+                    <td><%= weighted_label %></td><td><%= percent(entry[:row][primary_metric]) %></td>
+                    <td><%= escape(tokens(entry[:input_tokens])) %></td><td><%= escape(tokens(entry[:output_tokens])) %></td>
+                    <td class="standings-bar"><div class="track" role="meter" aria-valuemin="0" aria-valuemax="100" aria-valuenow="<%= (ranking * 100).round(1) %>"><i style="width:<%= (ranking * 100).clamp(0, 100) %>%"></i></div></td>
+                  </tr>
+                <% end %>
+              </tbody>
+            </table>
+          </div>
+        <% end %>
+
+        <h3 id="coverage">Coverage by model</h3>
+        <div class="scroll">
+          <table class="sortable">
+            <thead><tr><th>Provider</th><th>Model</th><th>Effort</th><th>Temperature</th><th>Scored</th><th>Weighted score</th><th>Errors</th><th>Cost</th></tr></thead>
+            <tbody>
+              <% standings[:entries].sort_by { [ranking_value(it).nil? ? 1 : 0, -(ranking_value(it) || 0), -it[:successful], it[:errors]] }.each do |entry| %>
+                <% provider, model = model_parts(entry[:model]) %>
+                <% score = entry[:complete] && ranking_value(entry) ? percent(ranking_value(entry)) : "Not rankable" %>
+                <tr>
+                  <td><span class="provider"><%= provider %></span></td><td><span class="model-id"><%= model %></span></td>
+                  <td><%= effort_label(entry[:model]) %></td><td><%= temperature_label(entry[:model]) %></td>
+                  <td><%= entry[:successful] %>/<%= entry[:total] %></td><td><%= score %></td>
+                  <td><%= escape(entry[:errors]) %></td><td><%= money(entry[:cost]) %></td>
+                </tr>
+              <% end %>
+            </tbody>
+          </table>
+        </div>
+      </section>
+
+      <% datasets.each_with_index do |dataset, index| %>
+        <% winner = dataset[:winner] %>
+        <% summary = if winner
+                       provider, model = model_parts(winner[:model])
+                       "Leader: #{provider}/#{model} · effort #{effort_label(winner[:model])} · temperature #{temperature_label(winner[:model])} · #{percent(ranking_value(winner))}"
+                     else
+                       "No complete contender"
+                     end %>
+        <details class="dataset" id="dataset-<%= index + 1 %>" open>
+          <summary><strong><%= escape(dataset[:name]) %></strong><span><%= summary %> · <%= dataset[:case_keys].length %> cases</span></summary>
+          <div class="dataset-content">
+            <h3>Model scores</h3>
+            <div class="scroll">
+              <table class="sortable">
+                <thead><tr><th>Provider</th><th>Model</th><th>Effort</th><th>Temperature</th><th>Weighted score</th><th>Scored</th><th>Errors</th><th title="Input tokens">↓ Tokens in</th><th title="Output tokens">↑ Tokens out</th><th>Cost</th></tr></thead>
+                <tbody>
+                  <% dataset[:entries].each do |entry| %>
+                    <% provider, model = model_parts(entry[:model]) %>
+                    <% score = entry[:complete] ? percent(ranking_value(entry)) : "Not rankable" %>
+                    <tr>
+                      <td><span class="provider"><%= provider %></span></td><td><span class="model-id"><%= model %></span></td>
+                      <td><%= effort_label(entry[:model]) %></td><td><%= temperature_label(entry[:model]) %></td>
+                      <td><%= score %></td><td><%= entry[:successful] %>/<%= entry[:total] %></td>
+                      <td><%= escape(entry[:errors]) %></td><td><%= escape(tokens(entry[:input_tokens])) %></td><td><%= escape(tokens(entry[:output_tokens])) %></td><td><%= money(entry[:cost]) %></td>
+                    </tr>
+                  <% end %>
+                </tbody>
+              </table>
+            </div>
+            <h3>Cases</h3>
+            <% dataset[:cases].each do |case_result| %>
+              <% kase = case_result.case %>
+              <% runs = case_results(kase, dataset[:summaries]) %>
+              <details class="case">
+                <summary><strong><%= escape(kase.id) %></strong><span><%= escape(kase.variant) %> · difficulty <%= escape(kase.difficulty) %></span></summary>
+                <div class="case-content">
+                  <details>
+                    <summary>Input and expected answer</summary>
+                    <h4>Input</h4><pre><%= escape(kase.input) %></pre>
+                    <h4>Expected</h4><pre><%= escape(kase.expected.nil? ? "No expected answer" : pretty(kase.expected)) %></pre>
+                  </details>
+                  <div class="scroll">
+                    <table class="sortable">
+                      <thead><tr><th>Provider</th><th>Model</th><th>Effort</th><th>Temperature</th><th>Score / status</th><th>Diagnostics</th></tr></thead>
+                      <tbody>
+                        <% runs.each do |run_result| %>
+                          <% provider, model = model_parts(run_result[:run].experiment.model) %>
+                          <tr>
+                            <td><span class="provider"><%= provider %></span></td><td><span class="model-id"><%= model %></span></td>
+                            <td><%= effort_label(run_result[:run].experiment.model) %></td><td><%= temperature_label(run_result[:run].experiment.model) %></td>
+                            <td><%= escape(run_result[:status]) %></td><td><pre><%= escape(pretty(run_result[:details])) %></pre></td>
+                          </tr>
+                        <% end %>
+                      </tbody>
+                    </table>
+                  </div>
+                </div>
+              </details>
+            <% end %>
+          </div>
+        </details>
+      <% end %>
+    </main>
+    <%= script_tag %>
+  </body>
+</html>

data/lib/llm/evals/templates/html_report.js

@@ -0,0 +1,48 @@
+document.querySelectorAll("table.sortable").forEach((table) => {
+  const headers = Array.from(table.querySelectorAll("thead th"));
+  const body = table.tBodies[0];
+
+  headers.forEach((header, index) => {
+    header.tabIndex = 0;
+    header.setAttribute("aria-sort", "none");
+
+    const sort = () => {
+      const ascending = header.dataset.direction !== "ascending";
+
+      headers.forEach((item) => {
+        item.dataset.direction = "";
+        item.setAttribute("aria-sort", "none");
+      });
+
+      header.dataset.direction = ascending ? "ascending" : "descending";
+      header.setAttribute("aria-sort", header.dataset.direction);
+
+      const rows = Array.from(body.rows);
+      rows.sort((left, right) => {
+        const first = left.cells[index]?.textContent.trim() ?? "";
+        const second = right.cells[index]?.textContent.trim() ?? "";
+        const number = (value) => {
+          const normalized = value.replace(/[,$%]/g, "");
+          return /^-?\d+(?:\.\d+)?$/.test(normalized) ? Number(normalized) : null;
+        };
+        const firstNumber = number(first);
+        const secondNumber = number(second);
+        const comparison = firstNumber !== null && secondNumber !== null
+          ? firstNumber - secondNumber
+          : first.localeCompare(second, undefined, { numeric: true, sensitivity: "base" });
+
+        return ascending ? comparison : -comparison;
+      });
+
+      rows.forEach((row) => body.append(row));
+    };
+
+    header.addEventListener("click", sort);
+    header.addEventListener("keydown", (event) => {
+      if (event.key === "Enter" || event.key === " ") {
+        event.preventDefault();
+        sort();
+      }
+    });
+  });
+});

data/lib/llm/evals/testing/echo_adapter.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Testing
+      # The smallest honest adapter: sends the case's input as the user
+      # prompt, parses the reply as a JSON object, and reports a reply that
+      # is not one as the model's failure. The toy suite runs on it, and a
+      # new suite can start from it.
+      class EchoAdapter < Adapter
+        SYSTEM = "Answer with a single JSON object and nothing else."
+        FENCE = /\A```(?:json)?\s*|\s*```\z/
+
+        # @param kase [Case]
+        # @param model [Model]
+        # @return [Reply]
+        def call(kase, model:)
+          completion = model.complete(system: SYSTEM, user: kase.input, json: true)
+          output = JSON.parse(completion.text.strip.gsub(FENCE, ""))
+          return Reply.new(error: "reply is not a JSON object", raw: completion.text, usage: completion.usage) unless output.is_a?(Hash)
+
+          Reply.new(output: output, raw: completion.text, usage: completion.usage)
+        rescue JSON::ParserError
+          Reply.new(error: "reply is not JSON", raw: completion.text, usage: completion.usage)
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/testing/fake_model.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Testing
+      # A {Model} on a {FakeProvider} of its own, for specs that run an
+      # adapter or an {Experiment} without the CLI.
+      class FakeModel < Model
+        # @param id [String]
+        # @param settings [Hash]
+        # @param provider_options [Hash] passed to {FakeProvider#initialize}
+        def initialize(id: "fake/echo", settings: { effort: nil, temperature: 0.2, max_tokens: 4096 }, **provider_options)
+          super(id: id, provider: FakeProvider.new(**provider_options), settings: settings)
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/testing/fake_provider.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Testing
+      # Serves every "fake/..." model without a network: a call answers with
+      # the expected JSON of the case whose input matches the user prompt, so
+      # a suite's own plumbing can be run end to end through the executable.
+      #
+      # `junk` names cases (by id or key) that get a non-JSON reply, and
+      # `wrong` maps a case to fields to answer wrongly. `answers`, a callable
+      # taking the user prompt, replaces the lookup altogether.
+      class FakeProvider < Providers::Base
+        PREFIX = "fake/"
+        USAGE = Usage.new(input_tokens: 100, output_tokens: 10, cost_usd: 0.001)
+        JUNK = "Sure! Here is my answer, in prose rather than JSON."
+
+        attr_reader :calls
+
+        # @param junk [Array<String>] case ids or keys to answer with prose
+        # @param wrong [Hash{String => Hash}] case id or key => fields to override
+        # @param answers [#call, nil] user prompt => reply text; replaces the lookup
+        # @param discovery [Discovery, nil] where to look cases up; the configured suite's when nil
+        def initialize(junk: [], wrong: {}, answers: nil, discovery: nil)
+          super()
+          @junk = junk
+          @wrong = wrong
+          @answers = answers
+          @discovery = discovery
+          @calls = []
+          @lock = Mutex.new
+        end
+
+        def name = "fake"
+        def request_type = :prompt
+        def prefix = PREFIX
+        def env_keys = []
+
+        # @return [nil] any id passes verification
+        def catalog = nil
+
+        # @param model [Model]
+        # @param system [String, nil]
+        # @param user [String]
+        # @param json [Boolean]
+        # @return [Completion]
+        def complete(model, system:, user:, json: true)
+          @lock.synchronize { @calls << { model: model.id, system: system, user: user, json: json } }
+          Completion.new(text: @answers ? @answers.call(user) : answer(user), usage: USAGE)
+        end
+
+        private
+
+        def answer(user)
+          kase = discovery.all.find { it.input == user }
+          return "{}" if kase.nil?
+          return JUNK if @junk.include?(kase.id) || @junk.include?(kase.key)
+
+          wrong = @wrong[kase.id] || @wrong[kase.key] || {}
+          JSON.generate(kase.expected.is_a?(Hash) ? kase.expected.merge(wrong) : kase.expected)
+        end
+
+        def discovery = @discovery || LLM::Evals.configuration.discovery
+      end
+    end
+  end
+end

data/lib/llm/evals/tracker.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # Where a run's results go besides the terminal. The gem always owns the
+    # run loop; a tracker only receives callbacks: {#start} once per
+    # experiment, {#record} once per finished case, {#finish} once with the
+    # summary. The local JSON log ({Trackers::Local}) always runs; at most
+    # one remote tracker runs beside it, chosen by `--tracker`, the
+    # `tracker:` key, or the environment (see {Trackers::Registry#choose}).
+    #
+    # A tracker for another framework (W&B Weave, an OTLP backend) subclasses
+    # this in a consumer or a plugin gem and is registered with `c.tracker`.
+    #
+    # @abstract Override the three callbacks; declare {.env_key} and {.gems}.
+    class Tracker
+      class << self
+        # @return [String] the name `--tracker` and `tracker:` use; the class name, dasherized, by default
+        def tracker_name = (name || "Tracker").split("::").last.gsub(/([a-z\d])([A-Z])/, '\1-\2').downcase
+
+        # @return [String, nil] the environment variable whose presence selects this tracker when nothing else does
+        def env_key = nil
+
+        # @return [Array<String>] the optional gems this tracker requires when chosen
+        def gems = []
+
+        # @param env [Hash] the environment
+        # @return [Boolean] whether the environment names this tracker
+        def detect?(env = ENV) = !env_key.nil? && !env[env_key].to_s.strip.empty?
+
+        # Loads the gems and checks the key, with an error that names both, so
+        # a chosen tracker that cannot run says so before any model is called.
+        #
+        # @param env [Hash]
+        # @return [void]
+        # @raise [MissingDependency, Error]
+        def requirements!(env = ENV)
+          gems.each do |gem_name|
+            LLM::Evals.optional!(gem_name.tr("-", "/"), gem_name, needed_for: "the #{tracker_name} tracker")
+          end
+          return if env_key.nil? || detect?(env)
+
+          raise Error, "the #{tracker_name} tracker needs #{env_key} in the environment (gems: #{gems.join(", ")})"
+        end
+      end
+
+      attr_reader :context
+
+      # @param context [Hash] `suite:`, `project:`, `project_id:` and `options:` (the tracker's own key in the config file)
+      def initialize(**context)
+        @context = context
+      end
+
+      # @return [String]
+      def name = self.class.tracker_name
+
+      # @return [Boolean] false turns every callback into a no-op
+      def enabled? = true
+
+      # @return [String] one line for `--verbose`: where this tracker sends things, and why
+      def describe = name
+
+      # @param experiment [Experiment]
+      # @return [void]
+      def start(_experiment) = nil
+
+      # @param result [RunResult]
+      # @return [void]
+      def record(_result) = nil
+
+      # @param experiment [Experiment]
+      # @param summary [Experiment::Summary]
+      # @return [Hash] `url:` to the experiment when one is known, else nil
+      def finish(_experiment, _summary) = { url: nil }
+
+      # @return [Hash] the tracker's own section of the config file
+      def options = context.fetch(:options, nil) || {}
+    end
+  end
+end

data/lib/llm/evals/trackers/braintrust.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Trackers
+      # Records each case as a Braintrust eval row in an experiment of the
+      # suite's project, through the braintrust gem's own OpenTelemetry
+      # exporter, the way `Braintrust::Eval.run` would have, except that the
+      # loop stays ours. The gem is required only when this tracker is chosen.
+      class Braintrust < Tracker
+        def self.env_key = "BRAINTRUST_API_KEY"
+        def self.gems = ["braintrust"]
+
+        attr_reader :experiment_id, :permalink
+
+        def describe = "braintrust: project #{context[:project_id] || context[:project].inspect}"
+
+        # Logs in, finds or creates the project, creates the experiment.
+        #
+        # @param experiment [Experiment]
+        # @return [void]
+        def start(experiment)
+          ::Braintrust.init(blocking_login: true, auto_instrument: false)
+          @state = ::Braintrust.current_state
+          @state.login
+          project_id = context[:project_id] || ::Braintrust::API::Internal::Projects.new(@state).create(name: context[:project]).fetch("id")
+          @experiment_id = ::Braintrust::API::Internal::Experiments.new(@state).create(
+            name:       experiment.name,
+            project_id: project_id,
+            ensure_new: true,
+            tags:       [context[:suite], experiment.model.id, "dataset-#{experiment.dataset}"].compact,
+            metadata:   metadata(experiment)
+          ).fetch("id")
+          @permalink = @state.object_permalink(object_type: "experiment", object_id: experiment_id)
+          # Braintrust.init installs its exporter on the global provider, which
+          # is where the gem's own runner takes its tracer from too.
+          @tracer = OpenTelemetry.tracer_provider.tracer("llm-evals")
+        end
+
+        # One root "eval" span per case, with the shape Braintrust's own
+        # runner emits: input, expected, output, scores and metadata as JSON
+        # attributes, parented to the experiment.
+        #
+        # @param result [RunResult]
+        # @return [void]
+        def record(result)
+          span = @tracer.start_root_span("eval")
+          span.set_attribute("braintrust.parent", "experiment_id:#{experiment_id}")
+          json(span, "braintrust.span_attributes", { type: "eval", name: result.experiment.name })
+          json(span, "braintrust.input_json", { case: result.case.key, text: result.case.input })
+          json(span, "braintrust.expected_json", result.case.expected) unless result.case.expected.nil?
+          json(span, "braintrust.metadata", case_metadata(result))
+          span.set_attribute("braintrust.tags", ["variant-#{result.case.variant}", "dataset-#{result.case.dataset}"])
+          if result.failed?
+            span.status = OpenTelemetry::Trace::Status.error(result.error)
+          else
+            json(span, "braintrust.output_json", result.reply.error? ? { "_error" => result.reply.error } : result.reply.output)
+            json(span, "braintrust.scores", result.score.metrics.compact)
+          end
+        ensure
+          span&.finish
+        end
+
+        # Flushes the spans and hands back the experiment's page.
+        #
+        # @return [Hash] `url:` the experiment permalink
+        def finish(_experiment, _summary)
+          provider = OpenTelemetry.tracer_provider
+          provider.force_flush if provider.respond_to?(:force_flush)
+          { url: permalink }
+        end
+
+        private
+
+        def json(span, key, value) = span.set_attribute(key, JSON.dump(value))
+
+        def metadata(experiment)
+          { model: experiment.model.id, provider: experiment.model.provider.name, dataset: experiment.dataset,
+            **experiment.model.settings, **experiment.adapter.metadata }
+        end
+
+        def case_metadata(result)
+          base = { case: result.case.key, dataset: result.case.dataset, variant: result.case.variant, difficulty: result.case.difficulty }
+          return base.merge(error: result.error) if result.failed?
+
+          base.merge(diagnostics: result.score.diagnostics, outcomes: result.score.outcomes, usage: result.reply.usage.to_h)
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/trackers/local.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module LLM
+  module Evals
+    module Trackers
+      # Always on: every experiment's results and summary land in one JSON
+      # file under `log/`, so a run is never lost to a closed terminal or a
+      # remote tracker that was never configured.
+      class Local < Tracker
+        attr_reader :dir, :path
+
+        # @param dir [String] where the files go; `log/llm-evals` at the repository root by default
+        # @param context [Hash] see {Tracker#initialize}
+        def initialize(dir: File.join(Dry::CLI::UI::Invocation.root(Dir.pwd), "log", "llm-evals"), **context)
+          super(**context)
+          @dir = dir
+          @results = []
+          @lock = Mutex.new
+        end
+
+        def describe = "local: #{dir}"
+
+        def start(experiment)
+          @started_at = Time.now
+          @path = File.join(dir, "#{slug(experiment.name)}.#{@started_at.strftime("%Y-%m-%d.%H%M%S")}.json")
+        end
+
+        def record(result)
+          @lock.synchronize { @results << result }
+        end
+
+        # @return [Hash] `url:` nil, `path:` the file written
+        def finish(experiment, summary)
+          FileUtils.mkdir_p(dir)
+          File.write(path, JSON.pretty_generate(document(experiment, summary)))
+          { url: nil, path: path }
+        end
+
+        private
+
+        def slug(text) = text.gsub(/[^A-Za-z0-9.-]+/, "_").squeeze("_").delete_prefix("_").delete_suffix("_")
+
+        def document(experiment, summary)
+          {
+            suite:      context[:suite],
+            experiment: experiment.name,
+            model:      experiment.model.id,
+            provider:   experiment.model.provider.name,
+            settings:   experiment.model.settings,
+            dataset:    experiment.dataset,
+            started_at: @started_at.iso8601,
+            duration:   summary.duration,
+            means:      summary.means,
+            errors:     summary.errors,
+            usage:      summary.usage.to_h,
+            results:    @lock.synchronize { @results.sort_by { experiment.cases.index(it.case) }.map { row(it) } }
+          }
+        end
+
+        def row(result)
+          {
+            case:        result.case.key,
+            duration:    result.duration,
+            error:       result.error,
+            score:       result.score&.value,
+            metrics:     result.score&.metrics,
+            outcomes:    result.score&.outcomes,
+            diagnostics: result.score&.diagnostics,
+            reply_error: result.reply&.error,
+            usage:       result.reply&.usage&.to_h
+          }
+        end
+      end
+    end
+  end
+end