phronomy 0.17.0 → 0.18.0

data/lib/phronomy/testing/eval/eval_case.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      EvalCase = Data.define(:input, :expected, :metadata) do
+        def initialize(input:, expected:, metadata: {})
+          super
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/eval_result.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      EvalResult = Data.define(:eval_case, :actual, :score, :usage, :latency_ms, :error) do
+        def initialize(eval_case:, actual:, score:, usage:, latency_ms:, error: nil)
+          super
+        end
+
+        def pass? = score >= 1.0
+        def scorer_error? = !error.nil?
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/metrics.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      class Metrics
+        def initialize(results)
+          @results = results
+        end
+
+        def pass_rate
+          return 0.0 if @results.empty?
+          @results.count(&:pass?).to_f / @results.size
+        end
+
+        def average_score
+          return 0.0 if @results.empty?
+          @results.sum(&:score) / @results.size
+        end
+
+        def total_usage
+          @results.map(&:usage).compact.reduce(Phronomy::TokenUsage.zero, :+)
+        end
+
+        def average_latency_ms
+          return 0.0 if @results.empty?
+          @results.sum(&:latency_ms).to_f / @results.size
+        end
+
+        def to_h
+          {
+            total: @results.size,
+            pass_count: @results.count(&:pass?),
+            pass_rate: pass_rate,
+            average_score: average_score,
+            total_usage: total_usage.to_h,
+            average_latency_ms: average_latency_ms
+          }
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/runner.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      # Sequential test-only evaluator. Concurrency belongs to the subsystem
+      # under test, not to the evaluation helper itself.
+      class Runner
+        def initialize(scorer: Scorer::ExactMatch.new)
+          @scorer = scorer
+        end
+
+        def run(dataset, callable)
+          dataset.to_a.map { |eval_case| run_one(eval_case, callable) }
+        end
+
+        private
+
+        def run_one(eval_case, callable)
+          started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+          result = callable.call(eval_case.input)
+          latency_ms = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond) - started_at
+          actual, usage = extract(result)
+          score, score_error = score_safely(
+            @scorer,
+            actual: actual,
+            expected: eval_case.expected,
+            input: eval_case.input
+          )
+          EvalResult.new(
+            eval_case: eval_case,
+            actual: actual,
+            score: score,
+            usage: usage,
+            latency_ms: latency_ms,
+            error: score_error
+          )
+        end
+
+        def extract(result)
+          result.is_a?(Hash) ? [result[:output].to_s, result[:usage]] : [result.to_s, nil]
+        end
+
+        def score_safely(scorer, **kwargs)
+          [scorer.score(**kwargs), nil]
+        rescue => error
+          [0.0, error]
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/scorer/base.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      module Scorer
+        class Base
+          def score(actual:, expected:, input: nil)
+            raise NotImplementedError, "#{self.class}#score is not implemented"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/scorer/exact_match.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      module Scorer
+        class ExactMatch < Base
+          def initialize(case_sensitive: true)
+            @case_sensitive = case_sensitive
+          end
+
+          def score(actual:, expected:, input: nil)
+            actual_value = actual.to_s.strip
+            expected_value = expected.to_s.strip
+            unless @case_sensitive
+              actual_value = actual_value.downcase
+              expected_value = expected_value.downcase
+            end
+            (actual_value == expected_value) ? 1.0 : 0.0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/scorer/includes_scorer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      module Scorer
+        class IncludesScorer < Base
+          def initialize(case_sensitive: false)
+            @case_sensitive = case_sensitive
+          end
+
+          def score(actual:, expected:, input: nil)
+            actual_value = actual.to_s
+            expected_value = expected.to_s
+            unless @case_sensitive
+              actual_value = actual_value.downcase
+              expected_value = expected_value.downcase
+            end
+            actual_value.include?(expected_value) ? 1.0 : 0.0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/scorer/llm_judge.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      module Scorer
+        class LlmJudge < Base
+          DEFAULT_PROMPT = <<~PROMPT
+            You are an impartial judge evaluating the quality of an AI assistant response.
+            Rate the response on a scale from 0.0 (completely wrong or unhelpful) to 1.0 (perfect).
+            Respond with ONLY a single decimal number between 0.0 and 1.0 — no other text.
+
+            Question: %<input>s
+            Expected answer: %<expected>s
+            Actual response: %<actual>s
+
+            Score:
+          PROMPT
+
+          def initialize(model:, prompt_template: DEFAULT_PROMPT, raise_on_error: false)
+            @model = model
+            @prompt_template = prompt_template
+            @raise_on_error = raise_on_error
+          end
+
+          def score(actual:, expected:, input: nil)
+            prompt = format(
+              @prompt_template,
+              input: input.to_s,
+              expected: expected.to_s,
+              actual: actual.to_s
+            )
+            response = Phronomy::Runtime.instance.offload.submit do
+              RubyLLM.chat(model: @model).ask(prompt)
+            end.blocking_wait
+            response.content.to_s.strip.scan(/-?\d+\.?\d*/).first.to_f.clamp(0.0, 1.0)
+          rescue => error
+            raise if @raise_on_error
+            warn "[LlmJudge] Scoring failed: #{error.message}"
+            0.0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/scorer.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      module Scorer
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    # Test-only evaluation helpers.
+    module Eval
+    end
+  end
+end

data/lib/phronomy/testing/fake_clock.rb

@@ -2,32 +2,16 @@
 
 module Phronomy
   module Testing
-    # A deterministic, manually-advanced clock for use in tests.
-    #
-    # Replaces real +Process.clock_gettime+ calls so that time-sensitive code
-    # can be tested without relying on wall-clock sleeps.
-    #
-    # @example
-    #   clock = Phronomy::Testing::FakeClock.new
-    #   clock.now        # => 0.0
-    #   clock.advance(5) # advance by 5 seconds
-    #   clock.now        # => 5.0
+    # Deterministic manually-advanced clock for tests.
     class FakeClock
-      # @return [Float] the current logical time in seconds since the epoch (t=0)
       attr_reader :now
 
       def initialize
         @now = 0.0
-        @callbacks = [] # [[fire_at, block], ...]
+        @callbacks = []
         @mutex = Mutex.new
       end
 
-      # Advance the clock by +seconds+ and fire any registered callbacks whose
-      # deadline has passed.
-      #
-      # @param seconds [Numeric]
-      # @return [self]
-      # @api private
       def advance(seconds)
         @mutex.synchronize do
           @now += seconds.to_f
@@ -36,65 +20,34 @@ module Phronomy
         self
       end
 
-      # Register a one-shot callback that fires when the clock reaches +at+.
-      #
-      # @param at [Numeric] logical time to fire
-      # @yield called with no arguments when the clock reaches +at+
-      # @return [self]
-      # @api private
       def at(at, &block)
         @mutex.synchronize { @callbacks << [at.to_f, block] }
         self
       end
 
-      # Schedule a one-shot callback to fire after +seconds+ from the current
-      # logical time.  This is the same interface as {Runtime::TimerQueue#schedule}
-      # so that a +FakeClock+ can be passed as a +timer_queue:+ argument in tests.
-      #
-      # @param seconds [Numeric] delay in logical seconds
-      # @yield called when the clock reaches the scheduled time
-      # @return [self]
-      # @api private
       def schedule(seconds:, &block)
         at(@now + seconds.to_f, &block)
       end
 
-      # Returns the number of pending (un-fired) callbacks.
-      # @return [Integer]
-      # @api private
       def pending_callbacks
         @mutex.synchronize { @callbacks.size }
       end
 
-      # Returns the logical time of the next pending callback, or +nil+ if
-      # there are no pending callbacks.
-      #
-      # @return [Float, nil]
-      # @api private
       def next_timer_at
         @mutex.synchronize { @callbacks.min_by { |(t, _)| t }&.first }
       end
 
-      # Advance the clock exactly to the next pending callback and fire it.
-      # Raises +RuntimeError+ when there are no pending callbacks.
-      #
-      # @return [self]
-      # @api private
       def advance_to_next_timer
         target = next_timer_at
         raise "No pending timers to advance to" unless target
-
         advance(target - @now)
       end
 
-      # Returns descriptive entries for all pending callbacks.
-      # Used by {Phronomy::Runtime::FakeScheduler#pending_timers}.
-      #
-      # @return [Array<Hash>] each entry: +{ fire_at:, description: nil }+
-      # @api private
       def pending_timer_entries
         @mutex.synchronize do
-          @callbacks.sort_by { |(t, _)| t }.map { |(t, _)| {fire_at: t, description: nil} }
+          @callbacks.sort_by { |(t, _)| t }.map do |(time, _)|
+            {fire_at: time, description: nil}
+          end
         end
       end
 
@@ -102,7 +55,7 @@ module Phronomy
 
       def fire_expired_callbacks!
         fired, @callbacks = @callbacks.partition { |(t, _)| t <= @now }
-        fired.sort_by { |(t, _)| t }.each { |(_, cb)| cb.call }
+        fired.sort_by { |(t, _)| t }.each { |(_, callback)| callback.call }
       end
     end
   end

data/lib/phronomy/testing.rb

@@ -1,12 +1,8 @@
 # frozen_string_literal: true
 
 module Phronomy
-  # Test helpers for deterministic, timer-independent testing.
-  #
-  # @example
-  #   require "phronomy/testing"
-  #   clock = Phronomy::Testing::FakeClock.new
-  #   scheduler = Phronomy::Testing::FakeScheduler.new
+  # Test-only helpers. Nothing under this namespace participates in the
+  # production Runtime control plane.
   module Testing
   end
 end

data/lib/phronomy/tools/agent.rb

@@ -2,11 +2,17 @@
 
 module Phronomy
   module Tools
-    # Wraps a Phronomy::Agent::Base subclass as a callable tool so that a parent
-    # agent can delegate a one-shot sub-task through the same stateful pipeline.
+    # Wraps a Phronomy::Agent::Base subclass as a callable Tool.
+    #
+    # Agent-backed Tools are logically asynchronous rather than offloaded
+    # synchronous operations. Their ToolInvocation starts the child Agent and then
+    # returns to EventLoop immediately; the Tool completion handle settles when the
+    # child Agent FSMSession finishes. An OffloadPool worker is therefore never
+    # consumed merely to wait for another Agent.
     class Agent < Phronomy::Agent::Context::Capability::Base
-      description "Wraps an agent as a tool"
-      param :input, type: :string, desc: "The input to forward to the wrapped agent"
+      execution_mode :cooperative
+      description "Wraps a Phronomy::Agent as a Tool"
+      param :input, type: :string, desc: "The input to forward to the wrapped Agent"
 
       class << self
         def from_agent(agent_class, tool_name: nil, description: nil)
@@ -25,13 +31,38 @@ module Phronomy
 
           klass.tool_name(effective_name)
           klass.description(effective_desc)
-          klass.define_method(:execute) do |input:|
+
+          # Preserve the synchronous Tool API for top-level callers. ToolInvocation
+          # never uses this path for Agent-backed Tools; it calls #call_async.
+          klass.define_method(:execute) do |input:, cancellation_token: nil|
+            invoke_options = {}
+            if cancellation_token
+              invoke_options[:config] = {cancellation_token: cancellation_token}
+            end
             result = Phronomy::Agent.run_once(
               definition: agent_class,
-              input: input
+              input: input,
+              **invoke_options
             )
             result[:output].to_s
           end
+
+          # Internal asynchronous execution protocol used by Agent#call_async.
+          # The child Agent owns its own FSMSession/EventLoop lifecycle; this
+          # method only returns its completion handle and performs a short map.
+          klass.define_method(:execute_async) do |input:, cancellation_token: nil, config: {}|
+            persistence = Phronomy::Persistence::InMemory.new
+            agent = agent_class.create(persistence: persistence)
+            task_config = (config || {}).dup
+            if cancellation_token && !task_config[:cancellation_token]
+              task_config[:cancellation_token] = cancellation_token
+            end
+
+            agent.invoke_async(input, config: task_config).map do |result|
+              result[:output].to_s
+            end
+          end
+          klass.send(:private, :execute_async)
           klass
         end
 
@@ -49,6 +80,110 @@ module Phronomy
             .sub(/_tool$/, "")
         end
       end
+
+      # Agent-backed Tools have an asynchronous implementation that does not use
+      # ToolExecutor/OffloadPool. Validation and Tool error policy still match
+      # Capability::Base#call.
+      def call_async(
+        args,
+        cancellation_token: nil,
+        config: {}
+      )
+        cancellation_token&.raise_if_cancelled!
+        validated_args, schema_error = send(:validate_and_coerce, args)
+
+        if schema_error
+          return schema_error_task(schema_error)
+        end
+
+        source = execute_async(
+          **(validated_args || {}),
+          cancellation_token: cancellation_token,
+          config: config || {}
+        )
+        unless source.respond_to?(:on_complete)
+          raise Phronomy::ToolError,
+            "#{self.class.name} asynchronous execution must return a completion handle"
+        end
+
+        result_task = Phronomy::Task.deferred(name: "agent-tool-#{name}")
+        source.on_complete do |result, error|
+          if error
+            settle_async_error(result_task, error)
+            next
+          end
+
+          begin
+            result_task.complete(send(:truncate_result_if_needed, result))
+          rescue => result_error
+            settle_async_error(result_task, result_error)
+          end
+        end
+        result_task
+      rescue Phronomy::ToolError, Phronomy::CancellationError => error
+        failed_task(error)
+      rescue => error
+        result_task = Phronomy::Task.deferred(name: "agent-tool-#{name}")
+        settle_async_error(result_task, error)
+        result_task
+      end
+
+      private
+
+      # Subclasses created by .from_agent and Orchestrator override this method.
+      # It deliberately remains private so it is not part of the public Tool API.
+      def execute_async(input:, cancellation_token: nil, config: {})
+        task = Phronomy::Task.deferred(name: "agent-tool-#{name}-fallback")
+        begin
+          task.complete(execute(input: input, cancellation_token: cancellation_token))
+        rescue => error
+          task.fail(error)
+        end
+        task
+      end
+
+      def schema_error_task(schema_error)
+        task = Phronomy::Task.deferred(name: "agent-tool-#{name}-schema")
+        if self.class.on_schema_error == :raise
+          task.fail(Phronomy::ToolError.new(
+            "#{self.class.name} schema error: #{schema_error}"
+          ))
+        else
+          task.complete("Schema validation failed: #{schema_error}")
+        end
+        task
+      end
+
+      def failed_task(error)
+        Phronomy::Task.deferred(name: "agent-tool-#{name}-failed").tap do |task|
+          task.fail(error)
+        end
+      end
+
+      def settle_async_error(task, error)
+        if error.is_a?(Phronomy::ToolError) || error.is_a?(Phronomy::CancellationError)
+          task.fail(error)
+          return task
+        end
+
+        if self.class.on_error == :suppress
+          message = "[Phronomy] Tool #{self.class.name} suppressed error: " \
+            "#{error.class}: #{error.message}"
+          if Phronomy.configuration.logger
+            Phronomy.configuration.logger.warn(message)
+          else
+            warn message
+          end
+          task.complete("Tool error suppressed: #{error.message}")
+        else
+          wrapped = Phronomy::ToolError.new(
+            "#{self.class.name} execution failed: #{error.message}"
+          )
+          wrapped.set_backtrace(error.backtrace)
+          task.fail(wrapped)
+        end
+        task
+      end
     end
   end
 end

data/lib/phronomy/vector_store/async_backend.rb

@@ -7,7 +7,7 @@ module Phronomy
     # Mixing this module into a VectorStore class provides three choices:
     #
     # 1. **Do nothing** — inherits default implementations from {VectorStore::Base}
-    #    that route through {BlockingAdapterPool} (the previous behaviour).
+    #    that route through {OffloadPool}.
     #
     # 2. **Override selectively** — override only the async methods where the
     #    backend has a native async driver, while the remaining methods fall back
@@ -30,7 +30,7 @@ module Phronomy
     module AsyncBackend
       # Async variant of {VectorStore::Base#add}.
       #
-      # Submits the add call to {BlockingAdapterPool} by default.
+      # Submits the add call to {OffloadPool} by default.
       # Override to use a native async driver.
       #
       # @param id                 [String]
@@ -38,12 +38,13 @@ module Phronomy
       # @param metadata           [Hash]
       # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
       # @param timeout            [Numeric, nil]
-      # @return [BlockingAdapterPool::PendingOperation]
+      # @return [OffloadPool::PendingOperation]
       # @api public
       def add_async(id:, embedding:, metadata: {}, cancellation_token: nil, timeout: nil)
-        Phronomy::Runtime.instance.blocking_io.submit(
+        Phronomy::Runtime.instance.offload.submit(
           timeout: timeout,
-          cancellation_token: cancellation_token
+          cancellation_token: cancellation_token,
+          on_full: :raise
         ) do
           add(id: id, embedding: embedding, metadata: metadata, cancellation_token: cancellation_token)
         end
@@ -51,19 +52,20 @@ module Phronomy
 
       # Async variant of {VectorStore::Base#search}.
       #
-      # Submits the search call to {BlockingAdapterPool} by default.
+      # Submits the search call to {OffloadPool} by default.
       # Override to use a native async driver.
       #
       # @param query_embedding    [Array<Float>]
       # @param k                  [Integer]
       # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
       # @param timeout            [Numeric, nil]
-      # @return [BlockingAdapterPool::PendingOperation]
+      # @return [OffloadPool::PendingOperation]
       # @api public
       def search_async(query_embedding:, k: 5, cancellation_token: nil, timeout: nil)
-        Phronomy::Runtime.instance.blocking_io.submit(
+        Phronomy::Runtime.instance.offload.submit(
           timeout: timeout,
-          cancellation_token: cancellation_token
+          cancellation_token: cancellation_token,
+          on_full: :raise
         ) do
           search(query_embedding: query_embedding, k: k, cancellation_token: cancellation_token)
         end
@@ -71,18 +73,19 @@ module Phronomy
 
       # Async variant of {VectorStore::Base#remove}.
       #
-      # Submits the remove call to {BlockingAdapterPool} by default.
+      # Submits the remove call to {OffloadPool} by default.
       # Override to use a native async driver.
       #
       # @param id                 [String]
       # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
       # @param timeout            [Numeric, nil]
-      # @return [BlockingAdapterPool::PendingOperation]
+      # @return [OffloadPool::PendingOperation]
       # @api public
       def remove_async(id:, cancellation_token: nil, timeout: nil)
-        Phronomy::Runtime.instance.blocking_io.submit(
+        Phronomy::Runtime.instance.offload.submit(
           timeout: timeout,
-          cancellation_token: cancellation_token
+          cancellation_token: cancellation_token,
+          on_full: :raise
         ) do
           remove(id: id)
         end
@@ -90,17 +93,18 @@ module Phronomy
 
       # Async variant of {VectorStore::Base#clear}.
       #
-      # Submits the clear call to {BlockingAdapterPool} by default.
+      # Submits the clear call to {OffloadPool} by default.
       # Override to use a native async driver.
       #
       # @param cancellation_token [Phronomy::Concurrency::CancellationToken, nil]
       # @param timeout            [Numeric, nil]
-      # @return [BlockingAdapterPool::PendingOperation]
+      # @return [OffloadPool::PendingOperation]
       # @api public
       def clear_async(cancellation_token: nil, timeout: nil)
-        Phronomy::Runtime.instance.blocking_io.submit(
+        Phronomy::Runtime.instance.offload.submit(
           timeout: timeout,
-          cancellation_token: cancellation_token
+          cancellation_token: cancellation_token,
+          on_full: :raise
         ) do
           clear
         end

data/lib/phronomy/vector_store/base.rb

@@ -9,8 +9,8 @@ module Phronomy
     #
     # Async methods (`search_async`, `add_async`, `remove_async`, `clear_async`)
     # are provided by the {AsyncBackend} mixin which defaults to routing calls
-    # through {BlockingAdapterPool}.  Backends with native async drivers may
-    # override individual async methods without touching the pool at all.
+    # through {OffloadPool}. Backends with native async drivers may override
+    # individual async methods without touching the pool at all.
     class Base
       include AsyncBackend