phronomy 0.16.0 → 0.18.0

data/lib/phronomy/eval/comparison.rb

@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    # Runs the same Dataset against two callables (configuration A and B) and
-    # packages the paired results for side-by-side comparison.
-    #
-    # @example
-    #   cmp     = Comparison.new(scorer: Scorer::ExactMatch.new)
-    #   pairs   = cmp.compare(dataset, callable_a, callable_b)
-    #   pairs.each do |pair|
-    #     puts "A=#{pair.result_a.score} B=#{pair.result_b.score}"
-    #   end
-    #
-    #   # Aggregate each side independently
-    #   Metrics.new(pairs.map(&:result_a)).to_h
-    class Comparison
-      # Holds the two EvalResults for a single EvalCase.
-      ComparisonPair = Data.define(:eval_case, :result_a, :result_b)
-
-      # @param scorer [Scorer::Base]
-      # @api public
-      def initialize(scorer: Scorer::ExactMatch.new)
-        @scorer = scorer
-      end
-
-      # Evaluates both callables on every case in the dataset.
-      #
-      # @param dataset    [Dataset]
-      # @param callable_a [#call]
-      # @param callable_b [#call]
-      # @return [Array<ComparisonPair>]
-      # @api public
-      def compare(dataset, callable_a, callable_b)
-        runner_a = Runner.new(scorer: @scorer)
-        runner_b = Runner.new(scorer: @scorer)
-
-        results_a = runner_a.run(dataset, callable_a)
-        results_b = runner_b.run(dataset, callable_b)
-
-        results_a.zip(results_b).map do |a, b|
-          ComparisonPair.new(eval_case: a.eval_case, result_a: a, result_b: b)
-        end
-      end
-    end
-  end
-end

data/lib/phronomy/eval/dataset.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    # An ordered collection of EvalCase objects.
-    #
-    # @example Build from a plain array of hashes
-    #   dataset = Dataset.from_array([
-    #     { input: "What is 2+2?", expected: "4" },
-    #     { input: "Capital of France?", expected: "Paris" }
-    #   ])
-    class Dataset
-      include Enumerable
-
-      # @param cases [Array<EvalCase>]
-      # @api public
-      def initialize(cases = [])
-        @cases = cases.freeze
-      end
-
-      # Constructs a Dataset from an Array of Hash-like objects.
-      # Each hash must have at least +:input+ and +:expected+ keys.
-      # An optional +:metadata+ key is forwarded as-is.
-      #
-      # @param pairs [Array<Hash>]
-      # @return [Dataset]
-      # @api public
-      def self.from_array(pairs)
-        new(pairs.map { |h| EvalCase.new(**h) })
-      end
-
-      # @yield [EvalCase]
-      # @api public
-      def each(&block)
-        @cases.each(&block)
-      end
-
-      # @return [Integer]
-      # @api public
-      def size
-        @cases.size
-      end
-    end
-  end
-end

data/lib/phronomy/eval/eval_case.rb

@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    # Represents a single evaluation sample with an input, an expected output,
-    # and optional freeform metadata.
-    #
-    # @example
-    #   EvalCase.new(input: "What is 2+2?", expected: "4")
-    #   EvalCase.new(input: "Hello", expected: "Hi", metadata: { difficulty: :easy })
-    EvalCase = Data.define(:input, :expected, :metadata) do
-      def initialize(input:, expected:, metadata: {})
-        super
-      end
-    end
-  end
-end

data/lib/phronomy/eval/eval_result.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    # An immutable record holding the outcome of evaluating one EvalCase.
-    #
-    # @!attribute eval_case  [EvalCase]  the original sample
-    # @!attribute actual     [String]    the callable's output
-    # @!attribute score      [Float]     scorer-assigned value in [0.0, 1.0]
-    # @!attribute usage      [Phronomy::TokenUsage, nil]
-    # @!attribute latency_ms [Integer]  wall-clock time of the callable in ms
-    # @!attribute error      [Exception, nil] set when the scorer raised an exception
-    EvalResult = Data.define(:eval_case, :actual, :score, :usage, :latency_ms, :error) do
-      def initialize(eval_case:, actual:, score:, usage:, latency_ms:, error: nil)
-        super
-      end
-
-      # Returns true when the scorer assigned a perfect score of 1.0.
-      def pass?
-        score >= 1.0
-      end
-
-      # Returns true when the scorer raised an exception.
-      def scorer_error?
-        !error.nil?
-      end
-    end
-  end
-end

data/lib/phronomy/eval/metrics.rb

@@ -1,66 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    # Aggregates a collection of EvalResult objects into summary statistics.
-    #
-    # @example
-    #   metrics = Metrics.new(results)
-    #   puts metrics.pass_rate        # => 0.8
-    #   puts metrics.average_score    # => 0.9
-    #   puts metrics.to_h
-    class Metrics
-      # @param results [Array<EvalResult>]
-      # @api public
-      def initialize(results)
-        @results = results
-      end
-
-      # Fraction of results that passed (score == 1.0).
-      # @return [Float] in [0.0, 1.0]
-      # @api public
-      def pass_rate
-        return 0.0 if @results.empty?
-        @results.count(&:pass?).to_f / @results.size
-      end
-
-      # Arithmetic mean of all scores.
-      # @return [Float]
-      # @api public
-      def average_score
-        return 0.0 if @results.empty?
-        @results.sum(&:score) / @results.size
-      end
-
-      # Sum of all TokenUsage objects present in the results.
-      # Results without usage are skipped.
-      # @return [Phronomy::TokenUsage]
-      # @api public
-      def total_usage
-        @results.map(&:usage).compact.reduce(TokenUsage.zero, :+)
-      end
-
-      # Arithmetic mean of latency_ms across all results.
-      # @return [Float]
-      # @api public
-      def average_latency_ms
-        return 0.0 if @results.empty?
-        @results.sum(&:latency_ms).to_f / @results.size
-      end
-
-      # Returns a plain Hash summary suitable for logging or serialisation.
-      # @return [Hash]
-      # @api public
-      def to_h
-        {
-          total: @results.size,
-          pass_count: @results.count(&:pass?),
-          pass_rate: pass_rate,
-          average_score: average_score,
-          total_usage: total_usage.to_h,
-          average_latency_ms: average_latency_ms
-        }
-      end
-    end
-  end
-end

data/lib/phronomy/eval/runner.rb

@@ -1,94 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    # Runs a Dataset through a callable and collects EvalResult objects.
-    #
-    # The callable must respond to +#call(input)+ and may return either:
-    # * a plain +String+ — treated as the output; usage is nil
-    # * a +Hash+ with +:output+ and optional +:usage+ (TokenUsage) keys
-    #
-    # @example With a simple proc
-    #   runner  = Runner.new(scorer: Scorer::ExactMatch.new)
-    #   dataset = Dataset.from_array([{ input: "2+2", expected: "4" }])
-    #   results = runner.run(dataset, ->(input) { "4" })
-    #
-    # @example With a Phronomy agent
-    #   agent   = MyAgent.new
-    #   results = runner.run(dataset, ->(input) { agent.invoke(input) })
-    class Runner
-      # @param scorer [Scorer::Base] scorer used to evaluate each result
-      # @api public
-      def initialize(scorer: Scorer::ExactMatch.new)
-        @scorer = scorer
-      end
-
-      # @param dataset     [Dataset]  collection of EvalCase objects
-      # @param callable    [#call]    accepts a single String argument
-      # @param concurrency [Integer]  number of parallel threads (default: 1, sequential)
-      # @return [Array<EvalResult>]
-      # @api public
-      # mutant:disable - concurrency default value mutations (0/2) are genuine equivalent because sequential and concurrent paths produce identical results; if concurrency<=1 boundary mutations (==1 / <1 / <=0 / .eql? / .equal? / false / nil / <=2) are genuine equivalent because the concurrent path with concurrency=1 still produces the same Array<EvalResult> via each_slice(1); spawn name: mutations are genuine equivalent (name is only used for logging)
-      def run(dataset, callable, concurrency: 1)
-        cases = dataset.to_a
-        return cases.map { |eval_case| run_one(eval_case, callable) } if concurrency <= 1
-
-        # Run cases in slices of +concurrency+ tasks. Each slice is joined
-        # before the next starts, bounding peak task count to +concurrency+.
-        # Writing to pre-allocated slots (one per task) is safe because each
-        # task writes to a unique index and all tasks in a slice are joined
-        # before the next slice begins.
-        # Exceptions in worker tasks are collected and re-raised after all
-        # tasks in the slice are joined, preventing orphaned tasks.
-        results = Array.new(cases.length)
-        cases.each_with_index.each_slice(concurrency) do |batch|
-          errors = []
-          errors_mu = Mutex.new
-          tasks = batch.map do |eval_case, i|
-            Phronomy::Runtime.instance.spawn(name: "eval-case-#{i}") do
-              results[i] = run_one(eval_case, callable)
-            rescue => e
-              errors_mu.synchronize { errors << e }
-            end
-          end
-          tasks.each(&:join)
-          raise errors.first if errors.any?
-        end
-        results
-      end
-
-      private
-
-      # Evaluate a single EvalCase with the given callable and return an EvalResult.
-      # mutant:disable - multiple genuine equivalent mutations: latency_ms=+t0 or =t0 are genuine because :millisecond makes all values Integer so be_a(Integer) passes; (actual,usage)=result is genuine because Ruby multi-assign of a String yields usage=nil identical to extract(); score_safely input: nil/eval_case/absent are genuine because ExactMatch and IncludesScorer ignore the :input kwarg; EvalResult error: nil/absent and usage: nil are genuine because on a successful score run score_error and usage are already nil
-      def run_one(eval_case, callable)
-        t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
-        result = callable.call(eval_case.input)
-        latency_ms = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond) - t0
-
-        actual, usage = extract(result)
-        score, score_error = score_safely(@scorer, actual: actual, expected: eval_case.expected, input: eval_case.input)
-
-        EvalResult.new(eval_case: eval_case, actual: actual, score: score, usage: usage, latency_ms: latency_ms, error: score_error)
-      end
-
-      # Normalises the callable's return value into [actual_string, usage_or_nil].
-      # mutant:disable - multiple genuine equivalent mutations: is_a?(Hash) vs instance_of?(Hash) (no Hash subclass in practice); to_s vs to_str (String only); result[:output]/[:usage] vs .fetch(:output)/[:usage] (keys always present when is_a?(Hash)); [result.to_s, nil] vs [result.to_s] because actual,usage=[val] → usage=nil via Ruby multi-assign; result.to_s vs result.to_str for String-only values
-      def extract(result)
-        if result.is_a?(Hash)
-          [result[:output].to_s, result[:usage]]
-        else
-          [result.to_s, nil]
-        end
-      end
-
-      # Calls the scorer and returns [score, error]. On failure, returns [0.0, exception].
-      # mutant:disable - [scorer.score(**kwargs), nil] vs [scorer.score(**kwargs)]: because score,error=[val] → error=nil via Ruby multi-assign; both produce the same destructuring in the caller
-      def score_safely(scorer, **kwargs)
-        [scorer.score(**kwargs), nil]
-      rescue => e
-        [0.0, e]
-      end
-    end
-  end
-end

data/lib/phronomy/eval/scorer/base.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    module Scorer
-      # Abstract base class for all scorers.
-      # Subclasses must implement {#score}.
-      class Base
-        # Scores an actual output against the expected output.
-        #
-        # @param actual   [String] the callable's output
-        # @param expected [String] the ground-truth value from the EvalCase
-        # @param input    [String, nil] the original input (used by LLM scorers)
-        # @return [Float] a value in [0.0, 1.0]
-        # @api public
-        def score(actual:, expected:, input: nil)
-          raise NotImplementedError, "#{self.class}#score is not implemented"
-        end
-      end
-    end
-  end
-end

data/lib/phronomy/eval/scorer/exact_match.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    module Scorer
-      # Scorer that returns 1.0 when the actual output exactly matches the
-      # expected output (after stripping leading/trailing whitespace).
-      # Comparison is case-sensitive by default.
-      #
-      # @example
-      #   ExactMatch.new.score(actual: "Paris", expected: "Paris")  # => 1.0
-      #   ExactMatch.new.score(actual: "paris", expected: "Paris")  # => 0.0
-      class ExactMatch < Base
-        # @param case_sensitive [Boolean] default true
-        # @api public
-        def initialize(case_sensitive: true)
-          @case_sensitive = case_sensitive
-        end
-
-        # @return [Float] 1.0 on match, 0.0 otherwise
-        # @api public
-        def score(actual:, expected:, input: nil)
-          a = actual.to_s.strip
-          e = expected.to_s.strip
-          a = a.downcase and e = e.downcase unless @case_sensitive
-          (a == e) ? 1.0 : 0.0
-        end
-      end
-    end
-  end
-end

data/lib/phronomy/eval/scorer/includes_scorer.rb

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    module Scorer
-      # Scorer that returns 1.0 when the actual output contains the expected
-      # substring (case-insensitive by default).
-      #
-      # Useful for open-ended outputs where the exact wording may vary but a
-      # key term or phrase must be present.
-      #
-      # @example
-      #   IncludesScorer.new.score(actual: "The answer is 42.", expected: "42")  # => 1.0
-      class IncludesScorer < Base
-        # @param case_sensitive [Boolean] default false
-        # @api public
-        def initialize(case_sensitive: false)
-          @case_sensitive = case_sensitive
-        end
-
-        # @return [Float] 1.0 if actual contains expected, 0.0 otherwise
-        # @api public
-        def score(actual:, expected:, input: nil)
-          a = actual.to_s
-          e = expected.to_s
-          a = a.downcase and e = e.downcase unless @case_sensitive
-          a.include?(e) ? 1.0 : 0.0
-        end
-      end
-    end
-  end
-end

data/lib/phronomy/eval/scorer/llm_judge.rb

@@ -1,72 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    module Scorer
-      # LLM-as-a-Judge scorer.
-      # Sends a structured prompt to an LLM and interprets its numeric reply
-      # as a quality score in [0.0, 1.0].
-      #
-      # The prompt template accepts three named placeholders:
-      #   %<input>s     — the original input question
-      #   %<expected>s  — the ground-truth / reference answer
-      #   %<actual>s    — the output being evaluated
-      #
-      # The LLM is expected to reply with a single decimal number; any extra
-      # text is stripped and the value is clamped to [0.0, 1.0].
-      # If parsing fails the scorer returns 0.0 rather than raising.
-      #
-      # @example
-      #   judge = LlmJudge.new(model: "gpt-4o-mini")
-      #   judge.score(actual: "Paris", expected: "Paris", input: "Capital of France?")
-      class LlmJudge < Base
-        DEFAULT_PROMPT = <<~PROMPT
-          You are an impartial judge evaluating the quality of an AI assistant response.
-          Rate the response on a scale from 0.0 (completely wrong or unhelpful) to 1.0 (perfect).
-          Respond with ONLY a single decimal number between 0.0 and 1.0 — no other text.
-
-          Question: %<input>s
-          Expected answer: %<expected>s
-          Actual response: %<actual>s
-
-          Score:
-        PROMPT
-
-        # @param model           [String]  RubyLLM model identifier
-        # @param prompt_template [String]  format string with %<input>s, %<expected>s, %<actual>s
-        # @param raise_on_error  [Boolean] when true, re-raises scoring exceptions instead of
-        #   returning 0.0. Use this in batch eval pipelines where silent failures are unacceptable.
-        # @api public
-        def initialize(model:, prompt_template: DEFAULT_PROMPT, raise_on_error: false)
-          @model = model
-          @prompt_template = prompt_template
-          @raise_on_error = raise_on_error
-        end
-
-        # @return [Float] score in [0.0, 1.0]; 0.0 on error when raise_on_error is false
-        # @api public
-        # mutant:disable - multiple genuine equivalent mutations:
-        #   actual.to_str / actual: (shorthand) are genuine (callers pass String);
-        #   expected.to_str / expected: are genuine (String);
-        #   response.content.strip (no to_s) is genuine (content is String);
-        #   lstrip/rstrip/no-strip are genuine (whitespace doesn't affect number scanning);
-        #   scan(/-?\d\.?\d*/) is genuine (for [0,1] range responses, single-digit-before-decimal
-        #     matches are the same after clamp);
-        #   response.content.to_str.strip is genuine (String);
-        #   all warn variations (warn no-arg, warn(nil), warn(e), warn(nil literal),
-        #     nil-replacing-warn, warn-deletion) are genuine because the rescue block
-        #     still returns 0.0 — warn is a side-effect not tested by value assertions
-        def score(actual:, expected:, input: nil)
-          prompt = format(@prompt_template, input: input.to_s, expected: expected.to_s, actual: actual.to_s)
-          response = Phronomy::Runtime.instance.blocking_io.submit { RubyLLM.chat(model: @model).ask(prompt) }.blocking_wait
-          response.content.to_s.strip.scan(/-?\d+\.?\d*/).first.to_f.clamp(0.0, 1.0)
-        rescue => e
-          raise if @raise_on_error
-
-          warn "[LlmJudge] Scoring failed: #{e.message}"
-          0.0
-        end
-      end
-    end
-  end
-end

data/lib/phronomy/eval/scorer.rb

@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    # Namespace module for scorer implementations.
-    module Scorer
-    end
-  end
-end

data/lib/phronomy/eval.rb

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  # Namespace module for the evaluation framework.
-  module Eval
-  end
-end

data/lib/phronomy/knowledge_source.rb

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  # KnowledgeSource provides the interface for supplying context region 3 (Knowledge)
-  # to the Context::Assembler.
-  #
-  # Each implementation returns an array of knowledge chunks via #fetch(query:).
-  # Each chunk is a Hash with :content (String) and :type (Symbol) keys.
-  # The Assembler wraps each chunk in an XML context tag before injecting it.
-  module KnowledgeSource
-  end
-end