RubyGems - ruby-skill-bench - Versions diffs - 1.1.0 → 1.2.0 - Mend

ruby-skill-bench 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (60) hide show

checksums.yaml +4 -4
data/README.md +166 -35
data/docs/architecture.md +3 -1
data/docs/first-eval-guide.md +7 -7
data/docs/testing-guide.md +1 -1
data/lib/skill_bench/agent/react_agent/loop_runner.rb +44 -9
data/lib/skill_bench/agent/react_agent/step.rb +7 -1
data/lib/skill_bench/cli/batch_result_printer.rb +45 -0
data/lib/skill_bench/cli/eval/eval_options.rb +4 -0
data/lib/skill_bench/cli/help_printer.rb +10 -2
data/lib/skill_bench/cli/init_command.rb +2 -1
data/lib/skill_bench/cli/result_printer.rb +1 -1
data/lib/skill_bench/cli/run_command.rb +47 -9
data/lib/skill_bench/cli/validate_command.rb +242 -0
data/lib/skill_bench/cli.rb +3 -0
data/lib/skill_bench/client.rb +43 -1
data/lib/skill_bench/clients/all.rb +2 -0
data/lib/skill_bench/clients/base_client.rb +12 -1
data/lib/skill_bench/clients/base_url_validator.rb +105 -0
data/lib/skill_bench/clients/provider_config.rb +34 -1
data/lib/skill_bench/clients/provider_schemas.rb +4 -0
data/lib/skill_bench/clients/providers/mistral.rb +47 -0
data/lib/skill_bench/commands/init.rb +5 -0
data/lib/skill_bench/commands/skill_new.rb +3 -1
data/lib/skill_bench/config/applier.rb +2 -0
data/lib/skill_bench/config/defaults.rb +2 -0
data/lib/skill_bench/config/facade_readers.rb +7 -0
data/lib/skill_bench/config/facade_writers.rb +17 -0
data/lib/skill_bench/config/json_loader.rb +1 -1
data/lib/skill_bench/config/store.rb +29 -0
data/lib/skill_bench/config.rb +18 -0
data/lib/skill_bench/evaluation/runner.rb +20 -3
data/lib/skill_bench/execution/context_hydrator.rb +52 -11
data/lib/skill_bench/execution/sandbox.rb +58 -11
data/lib/skill_bench/judge/judge.rb +4 -0
data/lib/skill_bench/judge/prompt.rb +42 -6
data/lib/skill_bench/models/config.rb +32 -0
data/lib/skill_bench/output_formatter.rb +60 -1
data/lib/skill_bench/package_verifier.rb +1 -1
data/lib/skill_bench/rails/skill_templates.rb +19 -5
data/lib/skill_bench/services/agent_spawner_service.rb +7 -3
data/lib/skill_bench/services/batch_runner_service.rb +111 -0
data/lib/skill_bench/services/compare_option_parser.rb +1 -0
data/lib/skill_bench/services/cost_calculator.rb +91 -0
data/lib/skill_bench/services/html_formatter.rb +289 -0
data/lib/skill_bench/services/json_formatter.rb +19 -1
data/lib/skill_bench/services/junit_formatter.rb +74 -24
data/lib/skill_bench/services/provider_resolver.rb +5 -2
data/lib/skill_bench/services/response_cache.rb +130 -0
data/lib/skill_bench/services/runner_service.rb +88 -4
data/lib/skill_bench/services/summary_formatter.rb +90 -0
data/lib/skill_bench/services/template_registry.rb +43 -9
data/lib/skill_bench/services/trend_recorder_service.rb +29 -2
data/lib/skill_bench/tools/registry.rb +29 -3
data/lib/skill_bench/tools/run_command.rb +171 -19
data/lib/skill_bench/trend_tracker/persistence.rb +27 -10
data/lib/skill_bench/trend_tracker.rb +5 -5
data/lib/skill_bench/version.rb +1 -1
data/lib/skill_bench.rb +2 -3
metadata +17 -36

data/lib/skill_bench/services/html_formatter.rb ADDED Viewed

@@ -0,0 +1,289 @@
+# frozen_string_literal: true
+require 'cgi'
+require_relative 'formatting_helpers'
+require_relative '../delta_report'
+module SkillBench
+  module Services
+    # Formats evaluation results as a complete, self-contained HTML document.
+    #
+    # The output embeds all styling inline (no external assets) and escapes every
+    # dynamic, user-derived value with {CGI.escapeHTML} to prevent HTML injection.
+    # Both the modern DeltaReport shape and the legacy result shape are supported.
+    class HtmlFormatter
+      extend FormattingHelpers
+      # Inline stylesheet embedded in every generated document.
+      STYLE = <<~CSS
+        body { font-family: -apple-system, Segoe UI, Roboto, sans-serif; margin: 2rem; color: #1a1a1a; background: #fafafa; }
+        main { max-width: 960px; margin: 0 auto; }
+        header { border-bottom: 2px solid #ddd; padding-bottom: 1rem; margin-bottom: 1.5rem; }
+        h1 { margin: 0 0 0.5rem; font-size: 1.6rem; }
+        dl.meta { display: grid; grid-template-columns: max-content 1fr; gap: 0.2rem 1rem; margin: 0.5rem 0; }
+        dl.meta dt { font-weight: 600; color: #555; }
+        dl.meta dd { margin: 0; }
+        p.usage { color: #555; font-variant-numeric: tabular-nums; }
+        table { border-collapse: collapse; width: 100%; margin: 1rem 0; }
+        th, td { padding: 0.4rem 0.75rem; text-align: right; border-bottom: 1px solid #e2e2e2; }
+        th:first-child, td:first-child { text-align: left; }
+        tr.total td { font-weight: 700; border-top: 2px solid #bbb; }
+        p.verdict { font-weight: 700; padding: 0.5rem 0.75rem; border-radius: 4px; display: inline-block; }
+        p.verdict.pass { background: #e6f4ea; color: #1e7e34; }
+        p.verdict.fail { background: #fde8e8; color: #c0392b; }
+        p.error { color: #c0392b; }
+        section.iterations h3 { margin-bottom: 0.25rem; }
+        ol { margin: 0.25rem 0 1rem; }
+        li { margin: 0.2rem 0; }
+        span.tools, span.observation { color: #555; }
+      CSS
+      # Format an eval result as a full HTML document.
+      #
+      # @param result [Hash] Eval result envelope (DeltaReport or legacy shape).
+      # @return [String] A complete HTML document string.
+      def self.format(result)
+        report = result.dig(:response, :report)
+        body = report.is_a?(SkillBench::DeltaReport) ? delta_body(result, report) : legacy_section(result)
+        build_document(result, body)
+      end
+      # Builds the body for a DeltaReport result (table plus iteration timeline).
+      #
+      # @param result [Hash] Eval result envelope.
+      # @param report [SkillBench::DeltaReport] The delta report.
+      # @return [String] HTML for the report and iteration sections.
+      def self.delta_body(result, report)
+        "#{report_section(report)}\n#{iterations_section(result)}"
+      end
+      private_class_method :delta_body
+      # Wraps body HTML in a complete, styled HTML document.
+      #
+      # @param result [Hash] Eval result envelope (used for the header/title).
+      # @param body [String] Pre-rendered body HTML.
+      # @return [String] A complete HTML document string.
+      def self.build_document(result, body)
+        title = escape(result[:eval_name] || 'Report')
+        <<~HTML
+          <!DOCTYPE html>
+          <html lang="en">
+          <head>
+          <meta charset="utf-8">
+          <title>SkillBench Report — #{title}</title>
+          <style>#{STYLE}</style>
+          </head>
+          <body>
+          <main>
+          #{header_html(result)}
+          #{body}
+          </main>
+          </body>
+          </html>
+        HTML
+      end
+      private_class_method :build_document
+      # Builds the header with eval/skill/provider names and the usage line.
+      #
+      # @param result [Hash] Eval result envelope.
+      # @return [String] HTML for the document header.
+      def self.header_html(result)
+        <<~HTML.chomp
+          <header>
+          <h1>SkillBench Report</h1>
+          <dl class="meta">
+          <dt>Eval</dt><dd>#{escape(result[:eval_name])}</dd>
+          <dt>Skill</dt><dd>#{escape(result[:skill_name])}</dd>
+          <dt>Provider</dt><dd>#{escape(result[:provider_name])}</dd>
+          </dl>
+          <p class="usage">#{usage_line(result)}</p>
+          </header>
+        HTML
+      end
+      private_class_method :header_html
+      # Builds the token/cost summary line for the header.
+      #
+      # @param result [Hash] Eval result envelope; reads :tokens and :cost.
+      # @return [String] An escaped "Tokens / Est. Cost" line.
+      def self.usage_line(result)
+        tokens = result[:tokens] || {}
+        total = tokens[:total_tokens] || tokens['total_tokens'] || 0
+        cost = result[:cost]
+        cost_label = cost ? Kernel.format('$%.4f', cost) : '—'
+        "Tokens: #{escape(total)} | Est. Cost: #{escape(cost_label)}"
+      end
+      private_class_method :usage_line
+      # Builds the scoring table and verdict for a DeltaReport.
+      #
+      # @param report [SkillBench::DeltaReport] The delta report.
+      # @return [String] HTML for the report section.
+      def self.report_section(report)
+        <<~HTML.chomp
+          <section class="report">
+          <h2>Delta Report</h2>
+          <table>
+          <thead><tr><th>Dimension</th><th>Baseline</th><th>Context</th><th>Delta</th></tr></thead>
+          <tbody>
+          #{dimension_rows(report)}
+          #{total_row(report)}
+          </tbody>
+          </table>
+          #{verdict_html(report)}
+          </section>
+        HTML
+      end
+      private_class_method :report_section
+      # Builds one table row per scored dimension.
+      #
+      # @param report [SkillBench::DeltaReport] The delta report.
+      # @return [String] HTML table rows joined by newlines.
+      def self.dimension_rows(report)
+        report.deltas.map { |name, delta| dimension_row(name, delta, report) }.join("\n")
+      end
+      private_class_method :dimension_rows
+      # Builds a single dimension table row.
+      #
+      # @param name [String] Dimension name.
+      # @param delta [Numeric] Context-minus-baseline delta.
+      # @param report [SkillBench::DeltaReport] The delta report.
+      # @return [String] An HTML table row.
+      def self.dimension_row(name, delta, report)
+        dim = report.criteria.dimensions.find { |candidate| candidate.name == name }
+        humanized = humanize(name)
+        label = dim ? "#{humanized} (#{dim.max_score})" : humanized
+        baseline = report.baseline_scores[name]
+        context = report.context_scores[name]
+        row_cells('dimension', label, baseline, context, delta_str(delta))
+      end
+      private_class_method :dimension_row
+      # Builds the totals table row.
+      #
+      # @param report [SkillBench::DeltaReport] The delta report.
+      # @return [String] An HTML table row for the totals.
+      def self.total_row(report)
+        total_delta = report.deltas.values.sum
+        row_cells('total', 'Total', "#{report.baseline_total}/100",
+                  "#{report.context_total}/100", delta_str(total_delta))
+      end
+      private_class_method :total_row
+      # Builds an HTML table row from escaped cell values.
+      #
+      # @param css_class [String] CSS class for the row.
+      # @param label [String] First-column label.
+      # @param baseline [Object] Baseline score cell.
+      # @param context [Object] Context score cell.
+      # @param delta [String] Delta cell.
+      # @return [String] An HTML table row.
+      def self.row_cells(css_class, label, baseline, context, delta)
+        "<tr class=\"#{css_class}\"><td>#{escape(label)}</td><td>#{escape(baseline)}</td>" \
+          "<td>#{escape(context)}</td><td>#{escape(delta)}</td></tr>"
+      end
+      private_class_method :row_cells
+      # Builds the verdict paragraph.
+      #
+      # @param report [SkillBench::DeltaReport] The delta report.
+      # @return [String] An HTML verdict paragraph.
+      def self.verdict_html(report)
+        verdict = report.verdict
+        criteria = report.criteria
+        status = verdict ? 'PASS' : 'FAIL'
+        css = verdict ? 'pass' : 'fail'
+        threshold = escape(criteria.pass_threshold)
+        minimum_delta = escape(criteria.minimum_delta)
+        %(<p class="verdict #{css}">Verdict: #{status} (threshold: #{threshold}, minimum delta: #{minimum_delta})</p>)
+      end
+      private_class_method :verdict_html
+      # Builds the baseline/context iteration timeline section.
+      #
+      # @param result [Hash] Eval result envelope.
+      # @return [String] HTML for the iterations section, or empty string.
+      def self.iterations_section(result)
+        baseline = result.dig(:response, :baseline_iterations) || []
+        context = result.dig(:response, :context_iterations) || []
+        baseline_empty = baseline.empty?
+        context_empty = context.empty?
+        return '' if baseline_empty && context_empty
+        blocks = []
+        blocks << iteration_block('Baseline Iterations', baseline) unless baseline_empty
+        blocks << iteration_block('Context Iterations', context) unless context_empty
+        %(<section class="iterations">\n<h2>Iteration Timeline</h2>\n#{blocks.join("\n")}\n</section>)
+      end
+      private_class_method :iterations_section
+      # Builds one named iteration timeline block.
+      #
+      # @param title [String] Section title.
+      # @param iterations [Array<Hash>] Iteration metadata entries.
+      # @return [String] HTML for the timeline block.
+      def self.iteration_block(title, iterations)
+        items = iterations.map { |iteration| iteration_item(iteration) }.join("\n")
+        %(<div class="timeline"><h3>#{escape(title)}</h3><ol>\n#{items}\n</ol></div>)
+      end
+      private_class_method :iteration_block
+      # Builds one list item for a single iteration step.
+      #
+      # @param iteration [Hash] Iteration metadata with :step_number, :thought,
+      #   :tools_used, and :observation_summary keys.
+      # @return [String] An HTML list item.
+      def self.iteration_item(iteration)
+        tools = iteration[:tools_used] || []
+        tools_html = tools.empty? ? '' : %( <span class="tools">Tools: #{escape(tools.join(', '))}</span>)
+        observation = iteration[:observation_summary].to_s
+        observation_html = observation.empty? ? '' : %( <span class="observation">Observation: #{escape(observation)}</span>)
+        step = "Step #{escape(iteration[:step_number])}: #{escape(iteration[:thought])}"
+        %(<li><span class="thought">#{step}</span>#{tools_html}#{observation_html}</li>)
+      end
+      private_class_method :iteration_item
+      # Builds the body for a legacy (non-DeltaReport) result.
+      #
+      # @param result [Hash] Legacy eval result envelope.
+      # @return [String] HTML for the legacy status section.
+      def self.legacy_section(result)
+        passed = result[:pass]
+        status = passed ? 'PASSED' : 'FAILED'
+        css = passed ? 'pass' : 'fail'
+        score = result[:score]&.round(2)
+        <<~HTML.chomp
+          <section class="report legacy">
+          <h2>Result</h2>
+          <p class="verdict #{css}">Status: #{status}</p>
+          <p class="score">Score: #{escape(score || 'N/A')}</p>
+          #{legacy_error(result)}
+          </section>
+        HTML
+      end
+      private_class_method :legacy_section
+      # Builds the optional error paragraph for a legacy result.
+      #
+      # @param result [Hash] Legacy eval result envelope.
+      # @return [String] An HTML error paragraph, or empty string.
+      def self.legacy_error(result)
+        message = result.dig(:response, :error, :message)
+        message ? %(<p class="error">Error: #{escape(message)}</p>) : ''
+      end
+      private_class_method :legacy_error
+      # Escapes any value for safe HTML embedding.
+      #
+      # @param value [Object] The value to escape (coerced via #to_s).
+      # @return [String] HTML-escaped text.
+      def self.escape(value)
+        CGI.escapeHTML(value.to_s)
+      end
+      private_class_method :escape
+    end
+  end
+end

data/lib/skill_bench/services/json_formatter.rb CHANGED Viewed

@@ -6,12 +6,30 @@ module SkillBench
   module Services
     # Formats evaluation results as JSON.
     class JsonFormatter
+      # Zeroed token usage used when a result carries no usage data.
+      EMPTY_USAGE = { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }.freeze
       # Format result as JSON.
       #
+      # Ensures top-level :tokens and :cost fields are always present (additive;
+      # existing keys are preserved) so JSON consumers see a stable shape.
+      #
       # @param result [Hash] Eval result.
       # @return [String] JSON-formatted string.
       def self.format(result)
-        JSON.pretty_generate(result)
+        JSON.pretty_generate(with_usage_fields(result))
+      end
+      # Returns the result augmented with token/cost fields when missing.
+      #
+      # @param result [Hash] Eval result (returned unchanged when not a Hash).
+      # @return [Hash] Result with :tokens and :cost guaranteed present.
+      def self.with_usage_fields(result)
+        return result unless result.is_a?(Hash)
+        tokens = result[:tokens] || result.dig(:response, :tokens) || EMPTY_USAGE
+        cost = result.key?(:cost) ? result[:cost] : result.dig(:response, :cost)
+        result.merge(tokens: tokens, cost: cost)
       end
     end
   end

data/lib/skill_bench/services/junit_formatter.rb CHANGED Viewed

@@ -4,39 +4,89 @@ require 'cgi'
 module SkillBench
   module Services
-    # Formats evaluation results as JUnit XML.
+    # Formats evaluation results as JUnit XML for CI consumption.
+    #
+    # Two entry points share the same per-result verdict/score logic:
+    # {.format} emits a single-result suite (one <testcase>), while
+    # {.format_batch} aggregates many results into one suite so a batch
+    # `skill-bench run --all` produces a single JUnit artifact.
     class JUnitFormatter
-      # Format result as JUnit XML.
+      # classname attribute applied to every emitted <testcase>.
+      CLASSNAME = 'SkillBench'
+      # Format a single result as a JUnit XML document.
       #
       # Supports both legacy format (result[:pass]) and modern DeltaReport format.
       #
       # @param result [Hash] Eval result.
       # @return [String] JUnit XML-formatted string.
       def self.format(result)
+        suite([result])
+      end
+      # Format an aggregate batch envelope as one JUnit XML document.
+      #
+      # Emits a single <testsuite> with one <testcase> per result, adding a
+      # <failure> child for every failing eval.
+      #
+      # @param aggregate [Hash] Aggregate envelope with a :results array.
+      # @return [String] JUnit XML-formatted string.
+      def self.format_batch(aggregate)
+        suite(aggregate[:results] || [])
+      end
+      # Builds a <testsuite> wrapping one <testcase> per result.
+      #
+      # @param results [Array<Hash>] Per-eval result envelopes.
+      # @return [String] JUnit XML-formatted string.
+      def self.suite(results)
+        failures = results.count { |result| !passing?(result) }
+        cases = results.map { |result| testcase(result) }.join("\n")
+        <<~XML
+          <?xml version="1.0"?>
+          <testsuite name="#{CLASSNAME}" tests="#{results.size}" failures="#{failures}">
+          #{cases}
+          </testsuite>
+        XML
+      end
+      private_class_method :suite
+      # Renders one <testcase> element (indented two spaces) for a result.
+      #
+      # @param result [Hash] A single-eval result envelope.
+      # @return [String] A <testcase> XML fragment.
+      def self.testcase(result)
+        name = CGI.escapeHTML(result[:eval_name].to_s)
+        return %(  <testcase name="#{name}" classname="#{CLASSNAME}"/>) if passing?(result)
+        score = CGI.escapeHTML(score_for(result).to_s)
+        [
+          %(  <testcase name="#{name}" classname="#{CLASSNAME}">),
+          %(    <failure message="Score: #{score}">Eval failed</failure>),
+          '  </testcase>'
+        ].join("\n")
+      end
+      private_class_method :testcase
+      # Whether a result passed (DeltaReport verdict or legacy :pass).
+      #
+      # @param result [Hash] A single-eval result envelope.
+      # @return [Boolean] true when the eval passed.
+      def self.passing?(result)
+        report = result.dig(:response, :report)
+        report.respond_to?(:verdict) ? report.verdict : result[:pass]
+      end
+      private_class_method :passing?
+      # The score reported for a failing eval.
+      #
+      # @param result [Hash] A single-eval result envelope.
+      # @return [Object] DeltaReport context_total or legacy :score.
+      def self.score_for(result)
         report = result.dig(:response, :report)
-        verdict = report.respond_to?(:verdict) ? report.verdict : result[:pass]
-        eval_name = CGI.escapeHTML(result[:eval_name].to_s)
-        if verdict
-          <<~XML
-            <?xml version="1.0"?>
-            <testsuite name="SkillBench" tests="1" failures="0">
-              <testcase name="#{eval_name}" classname="SkillBench"/>
-            </testsuite>
-          XML
-        else
-          score = report.respond_to?(:context_total) ? report.context_total : result[:score]
-          escaped_score = CGI.escapeHTML(score.to_s)
-          <<~XML
-            <?xml version="1.0"?>
-            <testsuite name="SkillBench" tests="1" failures="1">
-              <testcase name="#{eval_name}" classname="SkillBench">
-                <failure message="Score: #{escaped_score}">Eval failed</failure>
-              </testcase>
-            </testsuite>
-          XML
-        end
+        report.respond_to?(:context_total) ? report.context_total : result[:score]
       end
+      private_class_method :score_for
     end
   end
 end

data/lib/skill_bench/services/provider_resolver.rb CHANGED Viewed

@@ -51,11 +51,14 @@ module SkillBench
       private
       def resolve_provider
-        config = SkillBench::Models::Config.load
+        config = SkillBench::Models::Config.loaded
         provider = config.to_provider
         return provider if provider
-        warn 'Config load failed, using mock provider'
+        # Explicit `{"provider":"mock"}` is a valid choice, not a load failure,
+        # so it falls through to the mock provider without a warning. A missing
+        # provider key (genuine misconfiguration) still warns below.
+        warn 'Config load failed, using mock provider' unless config.mock?
         MOCK_PROVIDER.new('mock', 'mock', 'mock', {})
       rescue JSON::ParserError, ArgumentError, Errno::ENOENT => e
         # Config parsing/validation errors or missing config file - fall back to mock

data/lib/skill_bench/services/response_cache.rb ADDED Viewed

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+require 'digest'
+require 'json'
+module SkillBench
+  module Services
+    # Content-addressed, in-memory cache for LLM responses.
+    #
+    # The cache is opt-in and disabled by default. When enabled it lets repeated,
+    # identical LLM requests reuse a previously computed response instead of
+    # hitting the network again. The canonical example is `compare`, which runs
+    # the skill-less baseline twice with identical inputs.
+    #
+    # The backing store is a process-lifetime {Hash} keyed by a stable SHA-256
+    # digest of the request, so the same logical request always maps to the same
+    # entry regardless of hash-key ordering. Access to the store is serialized
+    # with a mutex so concurrent callers (e.g. {Parallel}-driven agents) cannot
+    # corrupt it or double-store a key.
+    class ResponseCache
+      # Environment variable that opts caching on when set to a truthy value.
+      ENV_FLAG = 'SKILL_BENCH_CACHE'
+      # Raw env values treated as "on".
+      TRUTHY_VALUES = %w[1 true yes on].freeze
+      # Guards every read/write of the shared store. Concurrent agents/judges run
+      # on separate threads; without this, the membership check and the write in
+      # {fetch} could interleave and store a key more than once.
+      MUTEX = Mutex.new
+      class << self
+        # Whether response caching is currently enabled.
+        #
+        # Enabled when {ENV_FLAG} is set to a truthy value (one of
+        # {TRUTHY_VALUES}); disabled when unset or set to anything else.
+        #
+        # @return [Boolean] true when caching is on
+        def enabled?
+          raw = ENV.fetch(ENV_FLAG, '').to_s.strip.downcase
+          TRUTHY_VALUES.include?(raw)
+        end
+        # Computes a stable content-addressed cache key for a request.
+        #
+        # The inputs are assembled into a canonical structure (hash keys sorted
+        # and stringified recursively) and hashed, so semantically identical
+        # requests always produce the same digest. Request-affecting provider
+        # configuration (endpoint/base URL/etc.) is included so two providers that
+        # share a name but target different endpoints never collide.
+        #
+        # @param provider [Symbol, String] Resolved provider identifier
+        # @param model [String, nil] Model name
+        # @param system_prompt [String] System prompt
+        # @param messages [Array<Hash>] Conversation messages
+        # @param tools [Array<Hash>, nil] Tool definitions, when present
+        # @param temperature [Float, nil] Sampling temperature, when present
+        # @param provider_config [Hash] Request-affecting provider settings such as
+        #   base_url, request_path, endpoint, location, project_id, api_version
+        # @return [String] Hex-encoded SHA-256 digest of the canonical request
+        def key(provider:, model:, system_prompt:, messages:, tools: nil, temperature: nil, provider_config: {})
+          payload = {
+            provider: provider.to_s,
+            model: model,
+            system_prompt: system_prompt,
+            messages: messages,
+            tools: tools,
+            temperature: temperature,
+            provider_config: provider_config
+          }
+          Digest::SHA256.hexdigest(JSON.generate(canonicalize(payload)))
+        end
+        # Returns the cached value for a key, computing and storing it on a miss.
+        #
+        # The value is computed outside the lock so requests for distinct keys run
+        # concurrently; the store read and the store write are each serialized by
+        # {MUTEX}, and a missing key is written exactly once (first writer wins).
+        #
+        # @param key [String] Cache key from {key}
+        # @yield Computes the value to cache when the key is absent
+        # @yieldreturn [Object] The value to cache
+        # @return [Object] The cached value (existing on a hit, freshly stored on a miss)
+        def fetch(key)
+          hit = MUTEX.synchronize { store[key] }
+          return hit unless hit.nil?
+          value = yield
+          MUTEX.synchronize { store[key] ||= value }
+        end
+        # Removes every cached entry.
+        #
+        # @return [void]
+        def clear
+          MUTEX.synchronize { store.clear }
+        end
+        private
+        # The process-lifetime backing store.
+        #
+        # @return [Hash{String => Object}] digest => cached response
+        def store
+          @store ||= {}
+        end
+        # Recursively rewrites a value into a stable form for serialization.
+        #
+        # Hashes get their keys stringified and sorted so that key ordering does
+        # not affect the resulting digest; arrays and scalars are preserved.
+        #
+        # @param value [Object] The value to canonicalize
+        # @return [Object] A canonical, order-stable copy of the value
+        def canonicalize(value)
+          case value
+          when Hash
+            value
+              .sort_by { |entry| entry.first.to_s }
+              .each_with_object({}) { |(name, val), acc| acc[name.to_s] = canonicalize(val) }
+          when Array
+            value.map { |element| canonicalize(element) }
+          else
+            value
+          end
+        end
+      end
+    end
+  end
+end