phronomy 0.17.0 → 0.18.0

data/lib/phronomy/eval/eval_case.rb

@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    # Represents a single evaluation sample with an input, an expected output,
-    # and optional freeform metadata.
-    #
-    # @example
-    #   EvalCase.new(input: "What is 2+2?", expected: "4")
-    #   EvalCase.new(input: "Hello", expected: "Hi", metadata: { difficulty: :easy })
-    EvalCase = Data.define(:input, :expected, :metadata) do
-      def initialize(input:, expected:, metadata: {})
-        super
-      end
-    end
-  end
-end

data/lib/phronomy/eval/eval_result.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    # An immutable record holding the outcome of evaluating one EvalCase.
-    #
-    # @!attribute eval_case  [EvalCase]  the original sample
-    # @!attribute actual     [String]    the callable's output
-    # @!attribute score      [Float]     scorer-assigned value in [0.0, 1.0]
-    # @!attribute usage      [Phronomy::TokenUsage, nil]
-    # @!attribute latency_ms [Integer]  wall-clock time of the callable in ms
-    # @!attribute error      [Exception, nil] set when the scorer raised an exception
-    EvalResult = Data.define(:eval_case, :actual, :score, :usage, :latency_ms, :error) do
-      def initialize(eval_case:, actual:, score:, usage:, latency_ms:, error: nil)
-        super
-      end
-
-      # Returns true when the scorer assigned a perfect score of 1.0.
-      def pass?
-        score >= 1.0
-      end
-
-      # Returns true when the scorer raised an exception.
-      def scorer_error?
-        !error.nil?
-      end
-    end
-  end
-end

data/lib/phronomy/eval/metrics.rb

@@ -1,66 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    # Aggregates a collection of EvalResult objects into summary statistics.
-    #
-    # @example
-    #   metrics = Metrics.new(results)
-    #   puts metrics.pass_rate        # => 0.8
-    #   puts metrics.average_score    # => 0.9
-    #   puts metrics.to_h
-    class Metrics
-      # @param results [Array<EvalResult>]
-      # @api public
-      def initialize(results)
-        @results = results
-      end
-
-      # Fraction of results that passed (score == 1.0).
-      # @return [Float] in [0.0, 1.0]
-      # @api public
-      def pass_rate
-        return 0.0 if @results.empty?
-        @results.count(&:pass?).to_f / @results.size
-      end
-
-      # Arithmetic mean of all scores.
-      # @return [Float]
-      # @api public
-      def average_score
-        return 0.0 if @results.empty?
-        @results.sum(&:score) / @results.size
-      end
-
-      # Sum of all TokenUsage objects present in the results.
-      # Results without usage are skipped.
-      # @return [Phronomy::TokenUsage]
-      # @api public
-      def total_usage
-        @results.map(&:usage).compact.reduce(TokenUsage.zero, :+)
-      end
-
-      # Arithmetic mean of latency_ms across all results.
-      # @return [Float]
-      # @api public
-      def average_latency_ms
-        return 0.0 if @results.empty?
-        @results.sum(&:latency_ms).to_f / @results.size
-      end
-
-      # Returns a plain Hash summary suitable for logging or serialisation.
-      # @return [Hash]
-      # @api public
-      def to_h
-        {
-          total: @results.size,
-          pass_count: @results.count(&:pass?),
-          pass_rate: pass_rate,
-          average_score: average_score,
-          total_usage: total_usage.to_h,
-          average_latency_ms: average_latency_ms
-        }
-      end
-    end
-  end
-end

data/lib/phronomy/eval/runner.rb

@@ -1,94 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    # Runs a Dataset through a callable and collects EvalResult objects.
-    #
-    # The callable must respond to +#call(input)+ and may return either:
-    # * a plain +String+ — treated as the output; usage is nil
-    # * a +Hash+ with +:output+ and optional +:usage+ (TokenUsage) keys
-    #
-    # @example With a simple proc
-    #   runner  = Runner.new(scorer: Scorer::ExactMatch.new)
-    #   dataset = Dataset.from_array([{ input: "2+2", expected: "4" }])
-    #   results = runner.run(dataset, ->(input) { "4" })
-    #
-    # @example With a Phronomy agent
-    #   agent   = MyAgent.new
-    #   results = runner.run(dataset, ->(input) { agent.invoke(input) })
-    class Runner
-      # @param scorer [Scorer::Base] scorer used to evaluate each result
-      # @api public
-      def initialize(scorer: Scorer::ExactMatch.new)
-        @scorer = scorer
-      end
-
-      # @param dataset     [Dataset]  collection of EvalCase objects
-      # @param callable    [#call]    accepts a single String argument
-      # @param concurrency [Integer]  number of parallel threads (default: 1, sequential)
-      # @return [Array<EvalResult>]
-      # @api public
-      # mutant:disable - concurrency default value mutations (0/2) are genuine equivalent because sequential and concurrent paths produce identical results; if concurrency<=1 boundary mutations (==1 / <1 / <=0 / .eql? / .equal? / false / nil / <=2) are genuine equivalent because the concurrent path with concurrency=1 still produces the same Array<EvalResult> via each_slice(1); spawn name: mutations are genuine equivalent (name is only used for logging)
-      def run(dataset, callable, concurrency: 1)
-        cases = dataset.to_a
-        return cases.map { |eval_case| run_one(eval_case, callable) } if concurrency <= 1
-
-        # Run cases in slices of +concurrency+ tasks. Each slice is joined
-        # before the next starts, bounding peak task count to +concurrency+.
-        # Writing to pre-allocated slots (one per task) is safe because each
-        # task writes to a unique index and all tasks in a slice are joined
-        # before the next slice begins.
-        # Exceptions in worker tasks are collected and re-raised after all
-        # tasks in the slice are joined, preventing orphaned tasks.
-        results = Array.new(cases.length)
-        cases.each_with_index.each_slice(concurrency) do |batch|
-          errors = []
-          errors_mu = Mutex.new
-          tasks = batch.map do |eval_case, i|
-            Phronomy::Runtime.instance.spawn(name: "eval-case-#{i}") do
-              results[i] = run_one(eval_case, callable)
-            rescue => e
-              errors_mu.synchronize { errors << e }
-            end
-          end
-          tasks.each(&:join)
-          raise errors.first if errors.any?
-        end
-        results
-      end
-
-      private
-
-      # Evaluate a single EvalCase with the given callable and return an EvalResult.
-      # mutant:disable - multiple genuine equivalent mutations: latency_ms=+t0 or =t0 are genuine because :millisecond makes all values Integer so be_a(Integer) passes; (actual,usage)=result is genuine because Ruby multi-assign of a String yields usage=nil identical to extract(); score_safely input: nil/eval_case/absent are genuine because ExactMatch and IncludesScorer ignore the :input kwarg; EvalResult error: nil/absent and usage: nil are genuine because on a successful score run score_error and usage are already nil
-      def run_one(eval_case, callable)
-        t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
-        result = callable.call(eval_case.input)
-        latency_ms = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond) - t0
-
-        actual, usage = extract(result)
-        score, score_error = score_safely(@scorer, actual: actual, expected: eval_case.expected, input: eval_case.input)
-
-        EvalResult.new(eval_case: eval_case, actual: actual, score: score, usage: usage, latency_ms: latency_ms, error: score_error)
-      end
-
-      # Normalises the callable's return value into [actual_string, usage_or_nil].
-      # mutant:disable - multiple genuine equivalent mutations: is_a?(Hash) vs instance_of?(Hash) (no Hash subclass in practice); to_s vs to_str (String only); result[:output]/[:usage] vs .fetch(:output)/[:usage] (keys always present when is_a?(Hash)); [result.to_s, nil] vs [result.to_s] because actual,usage=[val] → usage=nil via Ruby multi-assign; result.to_s vs result.to_str for String-only values
-      def extract(result)
-        if result.is_a?(Hash)
-          [result[:output].to_s, result[:usage]]
-        else
-          [result.to_s, nil]
-        end
-      end
-
-      # Calls the scorer and returns [score, error]. On failure, returns [0.0, exception].
-      # mutant:disable - [scorer.score(**kwargs), nil] vs [scorer.score(**kwargs)]: because score,error=[val] → error=nil via Ruby multi-assign; both produce the same destructuring in the caller
-      def score_safely(scorer, **kwargs)
-        [scorer.score(**kwargs), nil]
-      rescue => e
-        [0.0, e]
-      end
-    end
-  end
-end

data/lib/phronomy/eval/scorer/base.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    module Scorer
-      # Abstract base class for all scorers.
-      # Subclasses must implement {#score}.
-      class Base
-        # Scores an actual output against the expected output.
-        #
-        # @param actual   [String] the callable's output
-        # @param expected [String] the ground-truth value from the EvalCase
-        # @param input    [String, nil] the original input (used by LLM scorers)
-        # @return [Float] a value in [0.0, 1.0]
-        # @api public
-        def score(actual:, expected:, input: nil)
-          raise NotImplementedError, "#{self.class}#score is not implemented"
-        end
-      end
-    end
-  end
-end

data/lib/phronomy/eval/scorer/exact_match.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    module Scorer
-      # Scorer that returns 1.0 when the actual output exactly matches the
-      # expected output (after stripping leading/trailing whitespace).
-      # Comparison is case-sensitive by default.
-      #
-      # @example
-      #   ExactMatch.new.score(actual: "Paris", expected: "Paris")  # => 1.0
-      #   ExactMatch.new.score(actual: "paris", expected: "Paris")  # => 0.0
-      class ExactMatch < Base
-        # @param case_sensitive [Boolean] default true
-        # @api public
-        def initialize(case_sensitive: true)
-          @case_sensitive = case_sensitive
-        end
-
-        # @return [Float] 1.0 on match, 0.0 otherwise
-        # @api public
-        def score(actual:, expected:, input: nil)
-          a = actual.to_s.strip
-          e = expected.to_s.strip
-          a = a.downcase and e = e.downcase unless @case_sensitive
-          (a == e) ? 1.0 : 0.0
-        end
-      end
-    end
-  end
-end

data/lib/phronomy/eval/scorer/includes_scorer.rb

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    module Scorer
-      # Scorer that returns 1.0 when the actual output contains the expected
-      # substring (case-insensitive by default).
-      #
-      # Useful for open-ended outputs where the exact wording may vary but a
-      # key term or phrase must be present.
-      #
-      # @example
-      #   IncludesScorer.new.score(actual: "The answer is 42.", expected: "42")  # => 1.0
-      class IncludesScorer < Base
-        # @param case_sensitive [Boolean] default false
-        # @api public
-        def initialize(case_sensitive: false)
-          @case_sensitive = case_sensitive
-        end
-
-        # @return [Float] 1.0 if actual contains expected, 0.0 otherwise
-        # @api public
-        def score(actual:, expected:, input: nil)
-          a = actual.to_s
-          e = expected.to_s
-          a = a.downcase and e = e.downcase unless @case_sensitive
-          a.include?(e) ? 1.0 : 0.0
-        end
-      end
-    end
-  end
-end

data/lib/phronomy/eval/scorer/llm_judge.rb

@@ -1,72 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    module Scorer
-      # LLM-as-a-Judge scorer.
-      # Sends a structured prompt to an LLM and interprets its numeric reply
-      # as a quality score in [0.0, 1.0].
-      #
-      # The prompt template accepts three named placeholders:
-      #   %<input>s     — the original input question
-      #   %<expected>s  — the ground-truth / reference answer
-      #   %<actual>s    — the output being evaluated
-      #
-      # The LLM is expected to reply with a single decimal number; any extra
-      # text is stripped and the value is clamped to [0.0, 1.0].
-      # If parsing fails the scorer returns 0.0 rather than raising.
-      #
-      # @example
-      #   judge = LlmJudge.new(model: "gpt-4o-mini")
-      #   judge.score(actual: "Paris", expected: "Paris", input: "Capital of France?")
-      class LlmJudge < Base
-        DEFAULT_PROMPT = <<~PROMPT
-          You are an impartial judge evaluating the quality of an AI assistant response.
-          Rate the response on a scale from 0.0 (completely wrong or unhelpful) to 1.0 (perfect).
-          Respond with ONLY a single decimal number between 0.0 and 1.0 — no other text.
-
-          Question: %<input>s
-          Expected answer: %<expected>s
-          Actual response: %<actual>s
-
-          Score:
-        PROMPT
-
-        # @param model           [String]  RubyLLM model identifier
-        # @param prompt_template [String]  format string with %<input>s, %<expected>s, %<actual>s
-        # @param raise_on_error  [Boolean] when true, re-raises scoring exceptions instead of
-        #   returning 0.0. Use this in batch eval pipelines where silent failures are unacceptable.
-        # @api public
-        def initialize(model:, prompt_template: DEFAULT_PROMPT, raise_on_error: false)
-          @model = model
-          @prompt_template = prompt_template
-          @raise_on_error = raise_on_error
-        end
-
-        # @return [Float] score in [0.0, 1.0]; 0.0 on error when raise_on_error is false
-        # @api public
-        # mutant:disable - multiple genuine equivalent mutations:
-        #   actual.to_str / actual: (shorthand) are genuine (callers pass String);
-        #   expected.to_str / expected: are genuine (String);
-        #   response.content.strip (no to_s) is genuine (content is String);
-        #   lstrip/rstrip/no-strip are genuine (whitespace doesn't affect number scanning);
-        #   scan(/-?\d\.?\d*/) is genuine (for [0,1] range responses, single-digit-before-decimal
-        #     matches are the same after clamp);
-        #   response.content.to_str.strip is genuine (String);
-        #   all warn variations (warn no-arg, warn(nil), warn(e), warn(nil literal),
-        #     nil-replacing-warn, warn-deletion) are genuine because the rescue block
-        #     still returns 0.0 — warn is a side-effect not tested by value assertions
-        def score(actual:, expected:, input: nil)
-          prompt = format(@prompt_template, input: input.to_s, expected: expected.to_s, actual: actual.to_s)
-          response = Phronomy::Runtime.instance.blocking_io.submit { RubyLLM.chat(model: @model).ask(prompt) }.blocking_wait
-          response.content.to_s.strip.scan(/-?\d+\.?\d*/).first.to_f.clamp(0.0, 1.0)
-        rescue => e
-          raise if @raise_on_error
-
-          warn "[LlmJudge] Scoring failed: #{e.message}"
-          0.0
-        end
-      end
-    end
-  end
-end

data/lib/phronomy/eval/scorer.rb

@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Eval
-    # Namespace module for scorer implementations.
-    module Scorer
-    end
-  end
-end

data/lib/phronomy/eval.rb

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  # Namespace module for the evaluation framework.
-  module Eval
-  end
-end

data/lib/phronomy/testing/fake_scheduler.rb

@@ -1,104 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Testing
-    # A deterministic event dispatcher for use in tests.
-    #
-    # Wraps a {Thread::Queue} and dispatches events one at a time via {#tick}
-    # or drains all pending events via {#tick_until_idle}.  Tests can inspect
-    # queue depth and verify event ordering without wall-clock sleeps.
-    #
-    # @example
-    #   scheduler = Phronomy::Testing::FakeScheduler.new
-    #   scheduler.post(:a)
-    #   scheduler.post(:b)
-    #   scheduler.queue_depth   # => 2
-    #   scheduler.tick          # dispatches :a
-    #   scheduler.queue_depth   # => 1
-    #   scheduler.tick_until_idle
-    #   scheduler.dispatched    # => [:a, :b]
-    class FakeScheduler
-      # @return [Array] all events dispatched so far (in order)
-      attr_reader :dispatched
-
-      def initialize
-        @queue = Thread::Queue.new
-        @dispatched = []
-        @handlers = {}
-      end
-
-      # Enqueue an event for later dispatch.
-      #
-      # @param event [Object]
-      # @return [self]
-      # @api private
-      def post(event)
-        @queue.push(event)
-        self
-      end
-
-      # Dispatch the next queued event.
-      # Calls the registered handler (if any) and records the event.
-      # Returns the dispatched event, or +nil+ if the queue is empty.
-      #
-      # @return [Object, nil]
-      # @api private
-      def tick
-        return nil if @queue.empty?
-
-        event = begin
-          @queue.pop(true)
-        rescue
-          nil
-        end
-        return nil unless event
-
-        @dispatched << event
-        handler = @handlers[event.class] || @handlers[:any]
-        handler&.call(event)
-        event
-      end
-
-      # Dispatch events until the queue is empty.
-      # Bounded by +max_ticks+ to prevent infinite loops.
-      #
-      # @param max_ticks [Integer]
-      # @return [Integer] number of events dispatched
-      # @api private
-      def tick_until_idle(max_ticks: 1000)
-        count = 0
-        while !@queue.empty? && count < max_ticks
-          tick
-          count += 1
-        end
-        count
-      end
-
-      # Returns the number of events waiting to be dispatched.
-      # @return [Integer]
-      # @api private
-      def queue_depth
-        @queue.size
-      end
-
-      # Register a handler block for events of the given class.
-      # Use +:any+ to handle all event types.
-      #
-      # @param klass [Class, :any]
-      # @yield [event]
-      # @return [self]
-      # @api private
-      def on(klass, &block)
-        @handlers[klass] = block
-        self
-      end
-
-      # Returns true when the queue is empty.
-      # @return [Boolean]
-      # @api private
-      def idle?
-        @queue.empty?
-      end
-    end
-  end
-end

data/lib/phronomy/testing/scheduler_helpers.rb

@@ -1,68 +0,0 @@
-# frozen_string_literal: true
-
-module Phronomy
-  module Testing
-    # RSpec helper module that provides a deterministic {Runtime} backed by
-    # {Phronomy::Runtime::FakeScheduler}.
-    #
-    # Include this module in your RSpec describe/context blocks and call
-    # {#with_fake_scheduler} to run a block of code inside a fully
-    # synchronous, event-logged runtime.
-    #
-    # @example Basic usage (no clock)
-    #   include Phronomy::Testing::SchedulerHelpers
-    #
-    #   it "records completed events" do
-    #     with_fake_scheduler do |sched|
-    #       Phronomy::Runtime.instance.spawn(name: "my-task") { 42 }
-    #       expect(sched.event_log.map { |e| e[:type] }).to include(:completed)
-    #     end
-    #   end
-    #
-    # @example With a FakeClock
-    #   include Phronomy::Testing::SchedulerHelpers
-    #
-    #   it "surfaces pending timers" do
-    #     clock = Phronomy::Testing::FakeClock.new
-    #     with_fake_scheduler(clock: clock) do |sched|
-    #       clock.schedule(seconds: 5) { :fired }
-    #       expect(sched.pending_timers.first[:fire_at]).to eq(5.0)
-    #     end
-    #   end
-    module SchedulerHelpers
-      # Run +block+ with a {Phronomy::Runtime} that uses
-      # {Phronomy::Runtime::FakeScheduler}.
-      #
-      # The global runtime is replaced for the duration of the block and
-      # restored afterwards, whether the block raises or not.
-      #
-      # @param clock [Phronomy::Testing::FakeClock, nil]
-      #   Optional fake clock to inject into the scheduler for timer support
-      #   and event timestamping.
-      # @yield [scheduler, clock] the {Runtime::FakeScheduler} and the clock
-      # @return [Object] the return value of the block
-      # @api private
-      def with_fake_scheduler(clock: nil)
-        scheduler = Phronomy::Runtime::FakeScheduler.new
-        scheduler.clock = clock if clock
-        runtime = Phronomy::Runtime.new(scheduler: scheduler)
-        original = Phronomy::Runtime.default_if_initialized_for_test
-        Phronomy::Runtime.replace_default_for_test(runtime)
-        begin
-          yield scheduler, clock
-        ensure
-          result = nil
-          begin
-            result = runtime.shutdown
-          ensure
-            Phronomy::Runtime.restore_default_for_test(original)
-          end
-          unless result&.cleanup_complete?
-            raise Phronomy::RuntimeShutdownError,
-              "Temporary test Runtime did not shut down completely"
-          end
-        end
-      end
-    end
-  end
-end