phronomy 0.16.0 → 0.18.0

data/lib/phronomy/multi_agent/team_coordinator.rb

@@ -4,95 +4,34 @@ require "securerandom"
 
 module Phronomy
   module MultiAgent
-    # Implements the "Agent teams" coordination pattern (Anthropic blog, Pattern 3).
-    #
-    # @see https://claude.com/blog/multi-agent-coordination-patterns
-    #
-    # A coordinator LLM agent decomposes work into tasks and enqueues them
-    # dynamically via built-in tools. A fixed set of worker agents processes tasks
-    # sequentially — one task per worker per turn — carrying forward their
-    # conversation history across assignments to accumulate domain context over time.
-    #
-    # Workers are selected in sequence (the worker with the fewest accumulated
-    # messages is chosen by default). Task dispatch is synchronous; there is no
-    # concurrent or parallel execution.
-    #
-    # The coordinator is an {Agent::Base} subclass that has two built-in tools:
-    # - +enqueue_task+ — adds a task description to the queue
-    # - +finalize+     — signals that all tasks have been enqueued
-    #
-    # Worker persistence is implemented by passing each worker's accumulated
-    # +messages+ array back as a top-level +messages:+ argument on every subsequent
-    # +invoke+ call, so the LLM retains context across multiple task assignments.
-    #
-    # @example Basic usage
-    #   class MigrationTeam < Phronomy::MultiAgent::TeamCoordinator
-    #     coordinator_model        "claude-3-5-sonnet-20241022"
-    #     coordinator_instructions <<~INST
-    #       Analyze the request and enqueue one migration task per service.
-    #       Call enqueue_task for each service, then call finalize.
-    #     INST
-    #
-    #     pool size: 3, agent: MigrationAgent
-    #
-    #     aggregate do |assignments|
-    #       { reports: assignments.map { |a| { task: a[:task][:description], result: a[:result] } } }
-    #     end
-    #   end
-    #
-    #   result = MigrationTeam.new.invoke("Migrate all services to Rails 8")
+    # Coordinator/worker multi-agent pattern with persistent worker Agent state.
     class TeamCoordinator
-      # Holds per-worker context between task invocations.
-      # Worker persistence is implemented by carrying +messages+ forward on each
-      # successive +agent#invoke+ call as the top-level +messages:+ argument..
       WorkerState = Struct.new(
-        :index,    # Integer — 0-based worker index
-        :agent,    # Agent::Base instance
-        :messages, # Array  — accumulated conversation history
-        :status    # Symbol — :idle | :available | :done
+        :index,
+        :agent,
+        :transcript_size,
+        :status
       ) do
-        # Returns true when this worker is ready to accept the next task.
-        def available? = [:idle, :available].include?(status)
+        def available? = %i[idle available].include?(status)
       end
       private_constant :WorkerState
 
       class << self
-        # Sets the LLM model for the coordinator agent.
-        # Falls back to +Phronomy.configuration.default_model+ when not set.
-        #
-        # @param value [String, nil]
         # @api public
         def coordinator_model(value = nil)
           value ? @coordinator_model = value : @coordinator_model
         end
 
-        # Sets the system instructions for the coordinator agent.
-        # The prompt should direct the LLM to call +enqueue_task+ for each task
-        # and then call +finalize+ when all tasks are enqueued.
-        #
-        # @param value [String, nil]
         # @api public
         def coordinator_instructions(value = nil)
           value ? @coordinator_instructions = value : @coordinator_instructions
         end
 
-        # Sets the LLM provider for the coordinator agent.
-        # Required when using a custom +BASE_URL+ (e.g. LM Studio, Ollama, vLLM)
-        # so that RubyLLM does not attempt to resolve an unknown model name.
-        # Pass the same value as +LLMConfig::PROVIDER+ in your examples.
-        #
-        # @param value [Symbol, nil]
         # @api public
         def coordinator_provider(value = nil)
           value ? @coordinator_provider = value : @coordinator_provider
         end
 
-        # Configures the set of workers.
-        #
-        # @param size     [Integer] number of persistent worker instances (tasks are assigned sequentially)
-        # @param agent    [Class]   Agent::Base subclass used for all workers
-        # @param on_error [Symbol]  +:raise+ (default) propagates worker exceptions;
-        #                           +:skip+ records the failure and continues with remaining tasks
         # @api public
         def pool(size:, agent:, on_error: :raise)
           @pool_size = Integer(size)
@@ -100,57 +39,31 @@ module Phronomy
           @on_error = on_error
         end
 
-        # Customises the worker selection algorithm.
-        # The block receives an Array of available WorkerState objects and must
-        # return the one to assign the next task to.
-        # Default: worker with the fewest accumulated messages (round-robin-like).
-        #
-        # @yield [Array<WorkerState>] available workers
-        # @yieldreturn [WorkerState] the chosen worker
         # @api public
         def schedule(&block)
           @scheduler = block
         end
 
-        # Defines how task assignments are merged into the final return value.
-        # The block receives an Array of assignment Hashes:
-        #   { task: Hash, result: String|nil, worker: Integer, error: Exception|nil }
-        # When omitted, the raw assignments array is returned.
-        #
-        # @yield [Array<Hash>] all completed (and skipped) task assignments
         # @api public
         def aggregate(&block)
           @aggregator = block
         end
 
-        # @!visibility private
         def _coordinator_model = @coordinator_model
-        # @!visibility private
         def _coordinator_instructions = @coordinator_instructions
-        # @!visibility private
         def _coordinator_provider = @coordinator_provider
-        # @!visibility private
         def _pool_size = @pool_size || 1
-        # @!visibility private
         def _worker_agent = @worker_agent
-        # @!visibility private
         def _on_error = @on_error || :raise
-        # @!visibility private
         def _scheduler = @scheduler
-        # @!visibility private
         def _aggregator = @aggregator
       end
 
-      # Runs the full team coordination: coordinator generates tasks, workers
-      # process them sequentially, and the aggregate block merges the results.
-      #
-      # @param team_input [String, Hash] the high-level objective given to the coordinator
-      # @param config     [Hash]         reserved for future use
-      # @return [Object] the return value of the aggregate block, or the raw assignments Array
-      # @raise [ArgumentError] when +pool :agent+ has not been configured
       # @api public
       def invoke(team_input, config: {})
-        raise ArgumentError, "pool :agent must be configured before invoking" unless self.class._worker_agent
+        unless self.class._worker_agent
+          raise ArgumentError, "pool :agent must be configured before invoking"
+        end
 
         task_queue = []
         run_coordinator(team_input, task_queue)
@@ -158,26 +71,13 @@ module Phronomy
         finalize_result(assignments)
       end
 
-      # Streaming version of +invoke+. Yields a Hash event for each completed or
-      # failed task assignment.
-      #
-      # Yielded Hash keys:
-      #   :type   — +:task_completed+ or +:task_failed+
-      #   :worker — worker index (Integer)
-      #   :task   — the task Hash from the queue ({ id:, description:, metadata:, enqueued_at: })
-      #   :result — output string, or +nil+ on failure
-      #   :error  — Exception, or +nil+ on success
-      #
-      # @param team_input [String, Hash]
-      # @param config     [Hash]
-      # @yield [Hash] one event per completed/failed task
-      # @return [Object] same as +invoke+
-      # @raise [ArgumentError] when +pool :agent+ has not been configured
       # @api public
       def stream(team_input, config: {}, &block)
         return invoke(team_input, config: config) unless block
 
-        raise ArgumentError, "pool :agent must be configured before invoking" unless self.class._worker_agent
+        unless self.class._worker_agent
+          raise ArgumentError, "pool :agent must be configured before invoking"
+        end
 
         task_queue = []
         run_coordinator(team_input, task_queue)
@@ -187,23 +87,25 @@ module Phronomy
 
       private
 
-      # Phase 1: Run the coordinator LLM agent to populate task_queue.
       def run_coordinator(team_input, task_queue)
         coordinator = build_coordinator_agent(task_queue)
         input = team_input.is_a?(String) ? team_input : team_input.to_s
         coordinator.invoke(input)
       end
 
-      # Phase 2: Process tasks from the queue using the worker pool.
-      # Workers accumulate message history across assignments.
       def run_workers(task_queue, &event_block)
         pool_size = self.class._pool_size
         agent_class = self.class._worker_agent
         on_error = self.class._on_error
         scheduler = self.class._scheduler
 
-        workers = Array.new(pool_size) do |i|
-          WorkerState.new(index: i, agent: agent_class.new, messages: [], status: :idle)
+        workers = Array.new(pool_size) do |index|
+          WorkerState.new(
+            index: index,
+            agent: agent_class.new,
+            transcript_size: 0,
+            status: :idle
+          )
         end
 
         assignments = []
@@ -214,40 +116,45 @@ module Phronomy
           worker = scheduler ? scheduler.call(available) : default_scheduler(available)
 
           begin
-            # Worker agent retains conversation history in its Journal.
             result = worker.agent.invoke(task[:description])
-            worker.messages = Array(result[:messages])
+            worker.transcript_size = worker.agent.transcript.length
             worker.status = :available
-            entry = {task: task, result: result[:output], worker: worker.index, error: nil}
+            entry = {
+              task: task,
+              result: result[:output],
+              worker: worker.index,
+              error: nil
+            }
             assignments << entry
             event_block&.call(entry.merge(type: :task_completed))
-          rescue => e
+          rescue => error
             worker.status = :available
             raise unless on_error == :skip
 
-            entry = {task: task, result: nil, worker: worker.index, error: e}
+            entry = {
+              task: task,
+              result: nil,
+              worker: worker.index,
+              error: error
+            }
             assignments << entry
             event_block&.call(entry.merge(type: :task_failed))
           end
         end
 
-        workers.each { |w| w.status = :done }
+        workers.each { |worker| worker.status = :done }
         assignments
       end
 
-      # Phase 3: Apply the aggregate block (or return raw assignments).
       def finalize_result(assignments)
         aggregator = self.class._aggregator
         aggregator ? aggregator.call(assignments) : assignments
       end
 
-      # Default scheduler: assign to the worker with the fewest accumulated
-      # messages (promotes round-robin-like distribution across the pool).
       def default_scheduler(available_workers)
-        available_workers.min_by { |w| w.messages.size }
+        available_workers.min_by(&:transcript_size)
       end
 
-      # Build an anonymous coordinator Agent::Base with the two built-in tools.
       def build_coordinator_agent(task_queue)
         coordinator_model_val = self.class._coordinator_model
         coordinator_instructions_val = self.class._coordinator_instructions
@@ -260,34 +167,38 @@ module Phronomy
           model coordinator_model_val
           provider coordinator_provider_val if coordinator_provider_val
           instructions coordinator_instructions_val
-          tools enqueue_tool, finalize_tool
+          tools(enqueue_tool => nil, finalize_tool => nil)
         end
 
         coordinator_class.new
       end
 
-      # Builds the +enqueue_task+ tool. Each call appends a task Hash to task_queue.
       def build_enqueue_tool(task_queue)
         Class.new(Phronomy::Agent::Context::Capability::Base) do
           tool_name "enqueue_task"
           description "Add a task to the worker queue."
+          execution_mode :cooperative
           param :description, type: :string, desc: "What the worker agent should do"
           param :metadata, type: :string, desc: "Optional metadata", required: false
 
           define_method(:execute) do |description:, metadata: nil|
-            task = {id: task_queue.size + 1, description: description, metadata: metadata, enqueued_at: Time.now}
+            task = {
+              id: task_queue.size + 1,
+              description: description,
+              metadata: metadata,
+              enqueued_at: Time.now
+            }
             task_queue << task
             "Task ##{task[:id]} enqueued: #{description}"
           end
         end
       end
 
-      # Builds the +finalize+ tool. Signals to the coordinator LLM that all tasks
-      # have been enqueued; returns a confirmation string.
       def build_finalize_tool(task_queue)
         Class.new(Phronomy::Agent::Context::Capability::Base) do
           tool_name "finalize"
           description "Signal that task generation is complete. Call this after all tasks have been enqueued."
+          execution_mode :cooperative
           param :summary, type: :string, desc: "Brief summary of what was enqueued", required: false
 
           define_method(:execute) do |summary: ""|

data/lib/phronomy/testing/eval/comparison.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      class Comparison
+        ComparisonPair = Data.define(:eval_case, :result_a, :result_b)
+
+        def initialize(scorer: Scorer::ExactMatch.new)
+          @scorer = scorer
+        end
+
+        def compare(dataset, callable_a, callable_b)
+          results_a = Runner.new(scorer: @scorer).run(dataset, callable_a)
+          results_b = Runner.new(scorer: @scorer).run(dataset, callable_b)
+          results_a.zip(results_b).map do |a, b|
+            ComparisonPair.new(eval_case: a.eval_case, result_a: a, result_b: b)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/dataset.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      class Dataset
+        include Enumerable
+
+        def initialize(cases = [])
+          @cases = cases.freeze
+        end
+
+        def self.from_array(pairs)
+          new(pairs.map { |item| EvalCase.new(**item) })
+        end
+
+        def each(&block)
+          @cases.each(&block)
+        end
+
+        def size
+          @cases.size
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/eval_case.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      EvalCase = Data.define(:input, :expected, :metadata) do
+        def initialize(input:, expected:, metadata: {})
+          super
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/eval_result.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      EvalResult = Data.define(:eval_case, :actual, :score, :usage, :latency_ms, :error) do
+        def initialize(eval_case:, actual:, score:, usage:, latency_ms:, error: nil)
+          super
+        end
+
+        def pass? = score >= 1.0
+        def scorer_error? = !error.nil?
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/metrics.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      class Metrics
+        def initialize(results)
+          @results = results
+        end
+
+        def pass_rate
+          return 0.0 if @results.empty?
+          @results.count(&:pass?).to_f / @results.size
+        end
+
+        def average_score
+          return 0.0 if @results.empty?
+          @results.sum(&:score) / @results.size
+        end
+
+        def total_usage
+          @results.map(&:usage).compact.reduce(Phronomy::TokenUsage.zero, :+)
+        end
+
+        def average_latency_ms
+          return 0.0 if @results.empty?
+          @results.sum(&:latency_ms).to_f / @results.size
+        end
+
+        def to_h
+          {
+            total: @results.size,
+            pass_count: @results.count(&:pass?),
+            pass_rate: pass_rate,
+            average_score: average_score,
+            total_usage: total_usage.to_h,
+            average_latency_ms: average_latency_ms
+          }
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/runner.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      # Sequential test-only evaluator. Concurrency belongs to the subsystem
+      # under test, not to the evaluation helper itself.
+      class Runner
+        def initialize(scorer: Scorer::ExactMatch.new)
+          @scorer = scorer
+        end
+
+        def run(dataset, callable)
+          dataset.to_a.map { |eval_case| run_one(eval_case, callable) }
+        end
+
+        private
+
+        def run_one(eval_case, callable)
+          started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+          result = callable.call(eval_case.input)
+          latency_ms = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond) - started_at
+          actual, usage = extract(result)
+          score, score_error = score_safely(
+            @scorer,
+            actual: actual,
+            expected: eval_case.expected,
+            input: eval_case.input
+          )
+          EvalResult.new(
+            eval_case: eval_case,
+            actual: actual,
+            score: score,
+            usage: usage,
+            latency_ms: latency_ms,
+            error: score_error
+          )
+        end
+
+        def extract(result)
+          result.is_a?(Hash) ? [result[:output].to_s, result[:usage]] : [result.to_s, nil]
+        end
+
+        def score_safely(scorer, **kwargs)
+          [scorer.score(**kwargs), nil]
+        rescue => error
+          [0.0, error]
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/scorer/base.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      module Scorer
+        class Base
+          def score(actual:, expected:, input: nil)
+            raise NotImplementedError, "#{self.class}#score is not implemented"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/scorer/exact_match.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      module Scorer
+        class ExactMatch < Base
+          def initialize(case_sensitive: true)
+            @case_sensitive = case_sensitive
+          end
+
+          def score(actual:, expected:, input: nil)
+            actual_value = actual.to_s.strip
+            expected_value = expected.to_s.strip
+            unless @case_sensitive
+              actual_value = actual_value.downcase
+              expected_value = expected_value.downcase
+            end
+            (actual_value == expected_value) ? 1.0 : 0.0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/scorer/includes_scorer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      module Scorer
+        class IncludesScorer < Base
+          def initialize(case_sensitive: false)
+            @case_sensitive = case_sensitive
+          end
+
+          def score(actual:, expected:, input: nil)
+            actual_value = actual.to_s
+            expected_value = expected.to_s
+            unless @case_sensitive
+              actual_value = actual_value.downcase
+              expected_value = expected_value.downcase
+            end
+            actual_value.include?(expected_value) ? 1.0 : 0.0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/scorer/llm_judge.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      module Scorer
+        class LlmJudge < Base
+          DEFAULT_PROMPT = <<~PROMPT
+            You are an impartial judge evaluating the quality of an AI assistant response.
+            Rate the response on a scale from 0.0 (completely wrong or unhelpful) to 1.0 (perfect).
+            Respond with ONLY a single decimal number between 0.0 and 1.0 — no other text.
+
+            Question: %<input>s
+            Expected answer: %<expected>s
+            Actual response: %<actual>s
+
+            Score:
+          PROMPT
+
+          def initialize(model:, prompt_template: DEFAULT_PROMPT, raise_on_error: false)
+            @model = model
+            @prompt_template = prompt_template
+            @raise_on_error = raise_on_error
+          end
+
+          def score(actual:, expected:, input: nil)
+            prompt = format(
+              @prompt_template,
+              input: input.to_s,
+              expected: expected.to_s,
+              actual: actual.to_s
+            )
+            response = Phronomy::Runtime.instance.offload.submit do
+              RubyLLM.chat(model: @model).ask(prompt)
+            end.blocking_wait
+            response.content.to_s.strip.scan(/-?\d+\.?\d*/).first.to_f.clamp(0.0, 1.0)
+          rescue => error
+            raise if @raise_on_error
+            warn "[LlmJudge] Scoring failed: #{error.message}"
+            0.0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/scorer.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      module Scorer
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    # Test-only evaluation helpers.
+    module Eval
+    end
+  end
+end