ruby-skill-bench 1.0.1 → 1.2.0

data/lib/skill_bench/services/agent_spawner_service.rb

@@ -7,6 +7,9 @@ module SkillBench
   module Services
     # Spawns and executes LLM agents for evaluation.
     class AgentSpawnerService
+      # Zeroed token usage used when a run produces no usage data (e.g. mock, rescue).
+      EMPTY_USAGE = { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }.freeze
+
       # Spawns the LLM agent with the given system prompt.
       #
       # @param evaluation [SkillBench::Models::Eval] The eval being run
@@ -33,7 +36,7 @@ module SkillBench
       #
       # @return [Hash] Agent response with result, status, runtime, usage, raw_response, iterations
       def call
-        return { result: 'mock result', status: :success, iterations: [] } if @provider.name == 'mock'
+        return { result: 'mock result', status: :success, iterations: [], usage: EMPTY_USAGE } if @provider.name == 'mock'
 
         client_params = build_client_params
         max_iterations = @config&.[](:max_iterations) || @config&.[]('max_iterations') || 25
@@ -63,6 +66,7 @@ module SkillBench
           final_answer = agent_result.dig(:response, :content) || ''
           diff = Execution::Sandbox.capture_diff(sandbox.path)
           iterations = agent_result.dig(:response, :iterations) || []
+          usage = agent_result.dig(:response, :usage) || EMPTY_USAGE
 
           output = [final_answer, diff].reject(&:empty?).join("\n\n")
 
@@ -70,7 +74,7 @@ module SkillBench
             result: output,
             status: status,
             runtime: @provider.runtime,
-            usage: {},
+            usage: usage,
             raw_response: agent_result,
             iterations: iterations
           }
@@ -80,7 +84,7 @@ module SkillBench
           result: "Error: #{e.message}",
           status: :error,
           runtime: @provider.runtime,
-          usage: {},
+          usage: EMPTY_USAGE,
           raw_response: { error: e.message, backtrace: e.backtrace },
           iterations: []
         }

data/lib/skill_bench/services/batch_runner_service.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require 'parallel'
+require_relative 'runner_service'
+require_relative '../output_formatter'
+require_relative '../runner'
+
+module SkillBench
+  module Services
+    # Orchestrates running many evals in a single batch.
+    #
+    # Discovers every eval under a target directory and runs
+    # {RunnerService} over each, returning an aggregate envelope with
+    # per-eval results and a pass/fail summary.
+    #
+    # Discovery reuses {SkillBench::Runner.discover_task_dirs} but never
+    # routes through the deprecated {SkillBench::Task::Evaluator}: each eval
+    # is executed by the supported {RunnerService}.
+    class BatchRunnerService
+      # Default directory scanned for evals when none is supplied.
+      DEFAULT_EVALS_DIR = 'evals'
+
+      # Default batch-level thread count.
+      #
+      # Each {RunnerService.call} already runs its baseline and context
+      # agents concurrently (#26), so this is kept modest to bound nested
+      # thread usage (batch threads x per-eval threads).
+      DEFAULT_THREADS = 2
+
+      # Runs every eval discovered under +evals_dir+.
+      #
+      # @param skill_names [Array<String>] Names of the skills to apply to every eval
+      # @param evals_dir [String] Directory to scan for evals
+      # @param pack [String, nil] Optional pack name for registry-based skill resolution
+      # @param registry_manifest [String, nil] Optional path to registry.json manifest
+      # @param threads [Integer] Batch-level thread count
+      # @return [Hash] Aggregate envelope with :results and :summary
+      # @raise [ArgumentError] when no evals are found under +evals_dir+
+      def self.call(skill_names:, evals_dir: DEFAULT_EVALS_DIR, pack: nil, registry_manifest: nil, threads: DEFAULT_THREADS)
+        new(
+          skill_names: skill_names,
+          evals_dir: evals_dir,
+          pack: pack,
+          registry_manifest: registry_manifest,
+          threads: threads
+        ).call
+      end
+
+      # @param skill_names [Array<String>] Names of the skills
+      # @param evals_dir [String] Directory to scan for evals
+      # @param pack [String, nil] Optional pack name
+      # @param registry_manifest [String, nil] Optional registry.json path
+      # @param threads [Integer] Batch-level thread count
+      def initialize(skill_names:, evals_dir:, pack:, registry_manifest:, threads:)
+        @skill_names = skill_names
+        @evals_dir = evals_dir
+        @pack = pack
+        @registry_manifest = registry_manifest
+        @threads = threads
+      end
+
+      # Discovers the target evals and runs each through {RunnerService}.
+      #
+      # @return [Hash] Aggregate envelope with :results and :summary
+      # @raise [ArgumentError] when no evals are found under the directory
+      def call
+        eval_dirs = discover_eval_dirs
+        raise ArgumentError, "No evals found under #{evals_dir}" if eval_dirs.empty?
+
+        results = run_all(eval_dirs)
+        { results: results, summary: summarize(results) }
+      end
+
+      private
+
+      attr_reader :skill_names, :evals_dir, :pack, :registry_manifest, :threads
+
+      # Finds every eval directory under the configured root.
+      #
+      # @return [Array<Pathname>] Directories that contain a task.md
+      def discover_eval_dirs
+        SkillBench::Runner.discover_task_dirs(Pathname.new(evals_dir))
+      end
+
+      # Runs every eval directory through {RunnerService} concurrently.
+      #
+      # @param eval_dirs [Array<Pathname>] Discovered eval directories
+      # @return [Array<Hash>] Per-eval RunnerService results
+      def run_all(eval_dirs)
+        Parallel.map(eval_dirs, in_threads: threads) do |eval_dir|
+          RunnerService.call(
+            eval_name: eval_dir.to_s,
+            skill_names: skill_names,
+            pack: pack,
+            registry_manifest: registry_manifest
+          )
+        end
+      end
+
+      # Tallies pass/fail counts, reusing the single-eval exit-code logic.
+      #
+      # @param results [Array<Hash>] Per-eval results
+      # @return [Hash] Summary with :total, :passed and :failed counts
+      def summarize(results)
+        passed = results.count { |result| SkillBench::OutputFormatter.exit_code(result).zero? }
+        { total: results.size, passed: passed, failed: results.size - passed }
+      end
+    end
+  end
+end

data/lib/skill_bench/services/compare_option_parser.rb

@@ -44,6 +44,7 @@ module SkillBench
           opts.on('--variant-b SPEC', 'Second variant (e.g., "pack:hanami" or "/path/to/skill")') { |v| options[:variant_b] = v }
           opts.on('--eval PATH', 'Path to the eval directory') { |v| options[:eval] = v }
           opts.on('--format FORMAT', 'Output format (human, json)') { |v| options[:format] = v.to_sym }
+          opts.on('--cache', 'Enable content-addressed response caching') { ENV['SKILL_BENCH_CACHE'] = '1' }
           opts.on('-h', '--help', 'Prints this help') do
             puts opts
             raise SkillBench::HelpRequested

data/lib/skill_bench/services/cost_calculator.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Services
+    # Estimates the USD cost of an LLM run from token usage and a model name.
+    #
+    # Prices are approximate, drawn from public OpenAI/Anthropic pricing pages,
+    # and expressed in USD per 1,000 tokens. Provider pricing changes over time,
+    # so treat the result as a rough estimate and extend {PRICES} as needed.
+    class CostCalculator
+      # Approximate per-model prices in USD per 1,000 tokens.
+      # Keyed by a canonical model prefix; longer prefixes win on lookup so that
+      # dated variants (e.g. "claude-sonnet-4-20250514") resolve correctly.
+      # Source: public OpenAI and Anthropic pricing pages (approximate).
+      PRICES = {
+        'gpt-4o-mini' => { input: 0.00015, output: 0.0006 },
+        'gpt-4o' => { input: 0.005, output: 0.015 },
+        'gpt-4-turbo' => { input: 0.01, output: 0.03 },
+        'gpt-4' => { input: 0.03, output: 0.06 },
+        'gpt-3.5-turbo' => { input: 0.0005, output: 0.0015 },
+        'claude-opus-4' => { input: 0.015, output: 0.075 },
+        'claude-sonnet-4' => { input: 0.003, output: 0.015 },
+        'claude-3-5-sonnet' => { input: 0.003, output: 0.015 },
+        'claude-3-5-haiku' => { input: 0.0008, output: 0.004 },
+        'claude-3-opus' => { input: 0.015, output: 0.075 },
+        'claude-3-sonnet' => { input: 0.003, output: 0.015 },
+        'claude-3-haiku' => { input: 0.00025, output: 0.00125 }
+      }.freeze
+
+      # Token count that one priced unit of {PRICES} covers.
+      TOKENS_PER_UNIT = 1000.0
+
+      # Estimates the USD cost for a run.
+      #
+      # @param usage [Hash, nil] Token usage with :prompt_tokens and :completion_tokens.
+      # @param model [String, nil] The model name (e.g. "gpt-4o").
+      # @return [Float, nil] Estimated cost in USD, or nil when the model is unknown.
+      def self.call(usage:, model:)
+        new(usage, model).call
+      end
+
+      # @param usage [Hash, nil] Token usage hash.
+      # @param model [String, nil] The model name.
+      def initialize(usage, model)
+        @usage = usage || {}
+        @model = model
+      end
+
+      # Estimates the USD cost for the configured usage and model.
+      #
+      # @return [Float, nil] Estimated cost in USD, or nil when the model is unknown.
+      def call
+        price = price_for(@model)
+        return nil unless price
+
+        input_cost = units(:prompt_tokens) * price[:input]
+        output_cost = units(:completion_tokens) * price[:output]
+        (input_cost + output_cost).round(6)
+      end
+
+      private
+
+      # Finds the price entry for a model by longest matching name prefix.
+      #
+      # @param model [String, nil] The model name.
+      # @return [Hash, nil] Price entry with :input and :output, or nil when unknown.
+      def price_for(model)
+        key = model.to_s.downcase
+        return PRICES[key] if PRICES.key?(key)
+
+        PRICES.select { |name, _| key.start_with?(name) }.max_by { |name, _| name.length }&.last
+      end
+
+      # Converts a usage token count into priced 1K-token units.
+      #
+      # @param key [Symbol] The usage key to read.
+      # @return [Float] The number of priced units.
+      def units(key)
+        token_count(key) / TOKENS_PER_UNIT
+      end
+
+      # Reads a token count from the usage hash, tolerating string keys.
+      #
+      # @param key [Symbol] The usage key (e.g. :prompt_tokens).
+      # @return [Integer] The token count, or zero when absent.
+      def token_count(key)
+        (@usage[key] || @usage[key.to_s] || 0).to_i
+      end
+    end
+  end
+end

data/lib/skill_bench/services/html_formatter.rb

@@ -0,0 +1,289 @@
+# frozen_string_literal: true
+
+require 'cgi'
+require_relative 'formatting_helpers'
+require_relative '../delta_report'
+
+module SkillBench
+  module Services
+    # Formats evaluation results as a complete, self-contained HTML document.
+    #
+    # The output embeds all styling inline (no external assets) and escapes every
+    # dynamic, user-derived value with {CGI.escapeHTML} to prevent HTML injection.
+    # Both the modern DeltaReport shape and the legacy result shape are supported.
+    class HtmlFormatter
+      extend FormattingHelpers
+
+      # Inline stylesheet embedded in every generated document.
+      STYLE = <<~CSS
+        body { font-family: -apple-system, Segoe UI, Roboto, sans-serif; margin: 2rem; color: #1a1a1a; background: #fafafa; }
+        main { max-width: 960px; margin: 0 auto; }
+        header { border-bottom: 2px solid #ddd; padding-bottom: 1rem; margin-bottom: 1.5rem; }
+        h1 { margin: 0 0 0.5rem; font-size: 1.6rem; }
+        dl.meta { display: grid; grid-template-columns: max-content 1fr; gap: 0.2rem 1rem; margin: 0.5rem 0; }
+        dl.meta dt { font-weight: 600; color: #555; }
+        dl.meta dd { margin: 0; }
+        p.usage { color: #555; font-variant-numeric: tabular-nums; }
+        table { border-collapse: collapse; width: 100%; margin: 1rem 0; }
+        th, td { padding: 0.4rem 0.75rem; text-align: right; border-bottom: 1px solid #e2e2e2; }
+        th:first-child, td:first-child { text-align: left; }
+        tr.total td { font-weight: 700; border-top: 2px solid #bbb; }
+        p.verdict { font-weight: 700; padding: 0.5rem 0.75rem; border-radius: 4px; display: inline-block; }
+        p.verdict.pass { background: #e6f4ea; color: #1e7e34; }
+        p.verdict.fail { background: #fde8e8; color: #c0392b; }
+        p.error { color: #c0392b; }
+        section.iterations h3 { margin-bottom: 0.25rem; }
+        ol { margin: 0.25rem 0 1rem; }
+        li { margin: 0.2rem 0; }
+        span.tools, span.observation { color: #555; }
+      CSS
+
+      # Format an eval result as a full HTML document.
+      #
+      # @param result [Hash] Eval result envelope (DeltaReport or legacy shape).
+      # @return [String] A complete HTML document string.
+      def self.format(result)
+        report = result.dig(:response, :report)
+        body = report.is_a?(SkillBench::DeltaReport) ? delta_body(result, report) : legacy_section(result)
+        build_document(result, body)
+      end
+
+      # Builds the body for a DeltaReport result (table plus iteration timeline).
+      #
+      # @param result [Hash] Eval result envelope.
+      # @param report [SkillBench::DeltaReport] The delta report.
+      # @return [String] HTML for the report and iteration sections.
+      def self.delta_body(result, report)
+        "#{report_section(report)}\n#{iterations_section(result)}"
+      end
+      private_class_method :delta_body
+
+      # Wraps body HTML in a complete, styled HTML document.
+      #
+      # @param result [Hash] Eval result envelope (used for the header/title).
+      # @param body [String] Pre-rendered body HTML.
+      # @return [String] A complete HTML document string.
+      def self.build_document(result, body)
+        title = escape(result[:eval_name] || 'Report')
+        <<~HTML
+          <!DOCTYPE html>
+          <html lang="en">
+          <head>
+          <meta charset="utf-8">
+          <title>SkillBench Report — #{title}</title>
+          <style>#{STYLE}</style>
+          </head>
+          <body>
+          <main>
+          #{header_html(result)}
+          #{body}
+          </main>
+          </body>
+          </html>
+        HTML
+      end
+      private_class_method :build_document
+
+      # Builds the header with eval/skill/provider names and the usage line.
+      #
+      # @param result [Hash] Eval result envelope.
+      # @return [String] HTML for the document header.
+      def self.header_html(result)
+        <<~HTML.chomp
+          <header>
+          <h1>SkillBench Report</h1>
+          <dl class="meta">
+          <dt>Eval</dt><dd>#{escape(result[:eval_name])}</dd>
+          <dt>Skill</dt><dd>#{escape(result[:skill_name])}</dd>
+          <dt>Provider</dt><dd>#{escape(result[:provider_name])}</dd>
+          </dl>
+          <p class="usage">#{usage_line(result)}</p>
+          </header>
+        HTML
+      end
+      private_class_method :header_html
+
+      # Builds the token/cost summary line for the header.
+      #
+      # @param result [Hash] Eval result envelope; reads :tokens and :cost.
+      # @return [String] An escaped "Tokens / Est. Cost" line.
+      def self.usage_line(result)
+        tokens = result[:tokens] || {}
+        total = tokens[:total_tokens] || tokens['total_tokens'] || 0
+        cost = result[:cost]
+        cost_label = cost ? Kernel.format('$%.4f', cost) : '—'
+        "Tokens: #{escape(total)} | Est. Cost: #{escape(cost_label)}"
+      end
+      private_class_method :usage_line
+
+      # Builds the scoring table and verdict for a DeltaReport.
+      #
+      # @param report [SkillBench::DeltaReport] The delta report.
+      # @return [String] HTML for the report section.
+      def self.report_section(report)
+        <<~HTML.chomp
+          <section class="report">
+          <h2>Delta Report</h2>
+          <table>
+          <thead><tr><th>Dimension</th><th>Baseline</th><th>Context</th><th>Delta</th></tr></thead>
+          <tbody>
+          #{dimension_rows(report)}
+          #{total_row(report)}
+          </tbody>
+          </table>
+          #{verdict_html(report)}
+          </section>
+        HTML
+      end
+      private_class_method :report_section
+
+      # Builds one table row per scored dimension.
+      #
+      # @param report [SkillBench::DeltaReport] The delta report.
+      # @return [String] HTML table rows joined by newlines.
+      def self.dimension_rows(report)
+        report.deltas.map { |name, delta| dimension_row(name, delta, report) }.join("\n")
+      end
+      private_class_method :dimension_rows
+
+      # Builds a single dimension table row.
+      #
+      # @param name [String] Dimension name.
+      # @param delta [Numeric] Context-minus-baseline delta.
+      # @param report [SkillBench::DeltaReport] The delta report.
+      # @return [String] An HTML table row.
+      def self.dimension_row(name, delta, report)
+        dim = report.criteria.dimensions.find { |candidate| candidate.name == name }
+        humanized = humanize(name)
+        label = dim ? "#{humanized} (#{dim.max_score})" : humanized
+        baseline = report.baseline_scores[name]
+        context = report.context_scores[name]
+        row_cells('dimension', label, baseline, context, delta_str(delta))
+      end
+      private_class_method :dimension_row
+
+      # Builds the totals table row.
+      #
+      # @param report [SkillBench::DeltaReport] The delta report.
+      # @return [String] An HTML table row for the totals.
+      def self.total_row(report)
+        total_delta = report.deltas.values.sum
+        row_cells('total', 'Total', "#{report.baseline_total}/100",
+                  "#{report.context_total}/100", delta_str(total_delta))
+      end
+      private_class_method :total_row
+
+      # Builds an HTML table row from escaped cell values.
+      #
+      # @param css_class [String] CSS class for the row.
+      # @param label [String] First-column label.
+      # @param baseline [Object] Baseline score cell.
+      # @param context [Object] Context score cell.
+      # @param delta [String] Delta cell.
+      # @return [String] An HTML table row.
+      def self.row_cells(css_class, label, baseline, context, delta)
+        "<tr class=\"#{css_class}\"><td>#{escape(label)}</td><td>#{escape(baseline)}</td>" \
+          "<td>#{escape(context)}</td><td>#{escape(delta)}</td></tr>"
+      end
+      private_class_method :row_cells
+
+      # Builds the verdict paragraph.
+      #
+      # @param report [SkillBench::DeltaReport] The delta report.
+      # @return [String] An HTML verdict paragraph.
+      def self.verdict_html(report)
+        verdict = report.verdict
+        criteria = report.criteria
+        status = verdict ? 'PASS' : 'FAIL'
+        css = verdict ? 'pass' : 'fail'
+        threshold = escape(criteria.pass_threshold)
+        minimum_delta = escape(criteria.minimum_delta)
+        %(<p class="verdict #{css}">Verdict: #{status} (threshold: #{threshold}, minimum delta: #{minimum_delta})</p>)
+      end
+      private_class_method :verdict_html
+
+      # Builds the baseline/context iteration timeline section.
+      #
+      # @param result [Hash] Eval result envelope.
+      # @return [String] HTML for the iterations section, or empty string.
+      def self.iterations_section(result)
+        baseline = result.dig(:response, :baseline_iterations) || []
+        context = result.dig(:response, :context_iterations) || []
+        baseline_empty = baseline.empty?
+        context_empty = context.empty?
+        return '' if baseline_empty && context_empty
+
+        blocks = []
+        blocks << iteration_block('Baseline Iterations', baseline) unless baseline_empty
+        blocks << iteration_block('Context Iterations', context) unless context_empty
+        %(<section class="iterations">\n<h2>Iteration Timeline</h2>\n#{blocks.join("\n")}\n</section>)
+      end
+      private_class_method :iterations_section
+
+      # Builds one named iteration timeline block.
+      #
+      # @param title [String] Section title.
+      # @param iterations [Array<Hash>] Iteration metadata entries.
+      # @return [String] HTML for the timeline block.
+      def self.iteration_block(title, iterations)
+        items = iterations.map { |iteration| iteration_item(iteration) }.join("\n")
+        %(<div class="timeline"><h3>#{escape(title)}</h3><ol>\n#{items}\n</ol></div>)
+      end
+      private_class_method :iteration_block
+
+      # Builds one list item for a single iteration step.
+      #
+      # @param iteration [Hash] Iteration metadata with :step_number, :thought,
+      #   :tools_used, and :observation_summary keys.
+      # @return [String] An HTML list item.
+      def self.iteration_item(iteration)
+        tools = iteration[:tools_used] || []
+        tools_html = tools.empty? ? '' : %( <span class="tools">Tools: #{escape(tools.join(', '))}</span>)
+        observation = iteration[:observation_summary].to_s
+        observation_html = observation.empty? ? '' : %( <span class="observation">Observation: #{escape(observation)}</span>)
+        step = "Step #{escape(iteration[:step_number])}: #{escape(iteration[:thought])}"
+        %(<li><span class="thought">#{step}</span>#{tools_html}#{observation_html}</li>)
+      end
+      private_class_method :iteration_item
+
+      # Builds the body for a legacy (non-DeltaReport) result.
+      #
+      # @param result [Hash] Legacy eval result envelope.
+      # @return [String] HTML for the legacy status section.
+      def self.legacy_section(result)
+        passed = result[:pass]
+        status = passed ? 'PASSED' : 'FAILED'
+        css = passed ? 'pass' : 'fail'
+        score = result[:score]&.round(2)
+        <<~HTML.chomp
+          <section class="report legacy">
+          <h2>Result</h2>
+          <p class="verdict #{css}">Status: #{status}</p>
+          <p class="score">Score: #{escape(score || 'N/A')}</p>
+          #{legacy_error(result)}
+          </section>
+        HTML
+      end
+      private_class_method :legacy_section
+
+      # Builds the optional error paragraph for a legacy result.
+      #
+      # @param result [Hash] Legacy eval result envelope.
+      # @return [String] An HTML error paragraph, or empty string.
+      def self.legacy_error(result)
+        message = result.dig(:response, :error, :message)
+        message ? %(<p class="error">Error: #{escape(message)}</p>) : ''
+      end
+      private_class_method :legacy_error
+
+      # Escapes any value for safe HTML embedding.
+      #
+      # @param value [Object] The value to escape (coerced via #to_s).
+      # @return [String] HTML-escaped text.
+      def self.escape(value)
+        CGI.escapeHTML(value.to_s)
+      end
+      private_class_method :escape
+    end
+  end
+end

data/lib/skill_bench/services/json_formatter.rb

@@ -6,12 +6,30 @@ module SkillBench
   module Services
     # Formats evaluation results as JSON.
     class JsonFormatter
+      # Zeroed token usage used when a result carries no usage data.
+      EMPTY_USAGE = { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }.freeze
+
       # Format result as JSON.
       #
+      # Ensures top-level :tokens and :cost fields are always present (additive;
+      # existing keys are preserved) so JSON consumers see a stable shape.
+      #
       # @param result [Hash] Eval result.
       # @return [String] JSON-formatted string.
       def self.format(result)
-        JSON.pretty_generate(result)
+        JSON.pretty_generate(with_usage_fields(result))
+      end
+
+      # Returns the result augmented with token/cost fields when missing.
+      #
+      # @param result [Hash] Eval result (returned unchanged when not a Hash).
+      # @return [Hash] Result with :tokens and :cost guaranteed present.
+      def self.with_usage_fields(result)
+        return result unless result.is_a?(Hash)
+
+        tokens = result[:tokens] || result.dig(:response, :tokens) || EMPTY_USAGE
+        cost = result.key?(:cost) ? result[:cost] : result.dig(:response, :cost)
+        result.merge(tokens: tokens, cost: cost)
       end
     end
   end