phronomy 0.17.0 → 0.19.0

data/lib/phronomy/persistence/in_memory.rb

@@ -66,7 +66,7 @@ module Phronomy
 
         def save(agent_id, expected_revision:, root:)
           @owner.synchronize do
-            current = load(agent_id)
+            current = @owner.state[:agents].fetch(agent_id.to_s) { raise NotFoundError, "agent not found: #{agent_id}" }
             unless current.agent_revision == expected_revision
               raise ConflictError,
                 "agent revision conflict: expected #{expected_revision}, actual #{current.agent_revision}"
@@ -162,7 +162,7 @@ module Phronomy
 
         def save(execution_id, expected_revision:, execution:)
           @owner.synchronize do
-            current = load(execution_id)
+            current = @owner.state[:executions].fetch(execution_id.to_s) { raise NotFoundError, "execution not found: #{execution_id}" }
             unless current.execution_revision == expected_revision
               raise ConflictError,
                 "execution revision conflict: expected #{expected_revision}, actual #{current.execution_revision}"
@@ -209,31 +209,112 @@ module Phronomy
         end
       end
 
-      attr_reader :state
+      class WorkflowStates
+        def initialize(owner) = @owner = owner
+
+        def load(thread_id)
+          @owner.synchronize do
+            record = @owner.workflow_state_data[thread_id.to_s]
+            next nil unless record
+
+            {
+              snapshot: @owner.deep_dup_workflow_value(record.fetch(:snapshot)),
+              revision: record.fetch(:revision)
+            }.freeze
+          end
+        end
+
+        def save(thread_id, expected_revision:, snapshot:)
+          @owner.synchronize do
+            key = thread_id.to_s
+            current = @owner.workflow_state_data[key]
+            actual_revision = current&.fetch(:revision)
+            unless actual_revision == expected_revision
+              raise ConflictError,
+                "workflow state revision conflict for #{key}: " \
+                "expected #{expected_revision.inspect}, actual #{actual_revision.inspect}"
+            end
+
+            next_revision = actual_revision ? actual_revision + 1 : 1
+            @owner.workflow_state_data[key] = {
+              snapshot: @owner.deep_dup_workflow_value(snapshot),
+              revision: next_revision
+            }
+            next_revision
+          end
+        end
+
+        def delete(thread_id, expected_revision:)
+          @owner.synchronize do
+            key = thread_id.to_s
+            current = @owner.workflow_state_data[key]
+            actual_revision = current&.fetch(:revision)
+            unless actual_revision == expected_revision
+              raise ConflictError,
+                "workflow state revision conflict for #{key}: " \
+                "expected #{expected_revision.inspect}, actual #{actual_revision.inspect}"
+            end
+            @owner.workflow_state_data.delete(key)
+          end
+          nil
+        end
+      end
+
+      attr_reader :state, :workflow_state_data
 
       def initialize
         @monitor = Monitor.new
         @state = {contents: {}, agents: {}, journals: {}, executions: {}}
+        @workflow_state_data = {}
         @contents = Contents.new(self)
         @agents = Agents.new(self)
         @journals = Journals.new(self)
         @executions = Executions.new(self)
-        @activations = Phronomy::Agent::ActivationRegistry.new
-        super(contents: @contents, agents: @agents, journals: @journals,
-              executions: @executions, activations: @activations)
+        @workflow_states = WorkflowStates.new(self)
+        super(
+          contents: @contents,
+          agents: @agents,
+          journals: @journals,
+          executions: @executions,
+          workflow_states: @workflow_states
+        )
       end
 
       def capabilities
         {atomic_all: true, atomic_admission: true, optimistic_revision: true}.freeze
       end
 
+      def assert_agent_watermark!(agent_id:, agent_revision:, journal_position:)
+        synchronize do
+          stored = @state[:agents][agent_id.to_s]
+          unless stored
+            raise NotFoundError, "Agent not found: #{agent_id}"
+          end
+
+          if stored.agent_revision != agent_revision
+            raise ConflictError,
+              "agent revision conflict: expected #{agent_revision}, actual #{stored.agent_revision}"
+          end
+
+          actual_position = Array(@state[:journals][agent_id.to_s]).length
+          if actual_position != journal_position
+            raise ConflictError,
+              "journal position conflict: expected #{journal_position}, actual #{actual_position}"
+          end
+
+          true
+        end
+      end
+
       def transaction
         synchronize do
-          snapshot = Marshal.load(Marshal.dump(@state))
+          state_snapshot = Marshal.load(Marshal.dump(@state))
+          workflow_snapshot = deep_dup_workflow_value(@workflow_state_data)
           begin
             yield self
           rescue
-            @state = snapshot
+            @state.replace(state_snapshot)
+            @workflow_state_data.replace(workflow_snapshot)
             raise
           end
         end
@@ -242,6 +323,30 @@ module Phronomy
       def synchronize(&block)
         @monitor.synchronize(&block)
       end
+
+      # Workflow fields historically accepted ordinary Ruby values in the
+      # in-memory store. Keep that contract without forcing the Agent durable
+      # state Marshal snapshot to serialize arbitrary Workflow values.
+      def deep_dup_workflow_value(value)
+        case value
+        when Hash
+          value.each_with_object({}) do |(key, child), result|
+            result[deep_dup_workflow_value(key)] = deep_dup_workflow_value(child)
+          end
+        when Array
+          value.map { |child| deep_dup_workflow_value(child) }
+        when NilClass, Symbol, Integer, Float, TrueClass, FalseClass
+          value
+        else
+          return value if value.frozen?
+
+          begin
+            value.dup
+          rescue TypeError
+            value
+          end
+        end
+      end
     end
   end
 end

data/lib/phronomy/persistence.rb

@@ -6,14 +6,14 @@ module Phronomy
     class NotFoundError < Phronomy::Error; end
     class UnsupportedBackendError < Phronomy::Error; end
 
-    attr_reader :contents, :agents, :journals, :executions, :activations
+    attr_reader :contents, :agents, :journals, :executions, :workflow_states
 
-    def initialize(contents:, agents:, journals:, executions:, activations:)
+    def initialize(contents:, agents:, journals:, executions:, workflow_states:)
       @contents = contents
       @agents = agents
       @journals = journals
       @executions = executions
-      @activations = activations
+      @workflow_states = workflow_states
       validate_capabilities!
     end
 
@@ -25,6 +25,15 @@ module Phronomy
       raise UnsupportedBackendError, "#{self.class} does not provide atomic_all"
     end
 
+    # Verifies that a live Agent still owns the durable base it hydrated.
+    # Backends should implement this as a revision/position precondition check,
+    # not as a state reload returned to the caller.
+    # @api private
+    def assert_agent_watermark!(agent_id:, agent_revision:, journal_position:)
+      raise UnsupportedBackendError,
+        "#{self.class} does not provide Agent durable-watermark checks"
+    end
+
     private
 
     def validate_capabilities!

data/lib/phronomy/testing/eval/comparison.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      class Comparison
+        ComparisonPair = Data.define(:eval_case, :result_a, :result_b)
+
+        def initialize(scorer: Scorer::ExactMatch.new)
+          @scorer = scorer
+        end
+
+        def compare(dataset, callable_a, callable_b)
+          results_a = Runner.new(scorer: @scorer).run(dataset, callable_a)
+          results_b = Runner.new(scorer: @scorer).run(dataset, callable_b)
+          results_a.zip(results_b).map do |a, b|
+            ComparisonPair.new(eval_case: a.eval_case, result_a: a, result_b: b)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/dataset.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      class Dataset
+        include Enumerable
+
+        def initialize(cases = [])
+          @cases = cases.freeze
+        end
+
+        def self.from_array(pairs)
+          new(pairs.map { |item| EvalCase.new(**item) })
+        end
+
+        def each(&block)
+          @cases.each(&block)
+        end
+
+        def size
+          @cases.size
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/eval_case.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      EvalCase = Data.define(:input, :expected, :metadata) do
+        def initialize(input:, expected:, metadata: {})
+          super
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/eval_result.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      EvalResult = Data.define(:eval_case, :actual, :score, :usage, :latency_ms, :error) do
+        def initialize(eval_case:, actual:, score:, usage:, latency_ms:, error: nil)
+          super
+        end
+
+        def pass? = score >= 1.0
+        def scorer_error? = !error.nil?
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/metrics.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      class Metrics
+        def initialize(results)
+          @results = results
+        end
+
+        def pass_rate
+          return 0.0 if @results.empty?
+          @results.count(&:pass?).to_f / @results.size
+        end
+
+        def average_score
+          return 0.0 if @results.empty?
+          @results.sum(&:score) / @results.size
+        end
+
+        def total_usage
+          @results.map(&:usage).compact.reduce(Phronomy::TokenUsage.zero, :+)
+        end
+
+        def average_latency_ms
+          return 0.0 if @results.empty?
+          @results.sum(&:latency_ms).to_f / @results.size
+        end
+
+        def to_h
+          {
+            total: @results.size,
+            pass_count: @results.count(&:pass?),
+            pass_rate: pass_rate,
+            average_score: average_score,
+            total_usage: total_usage.to_h,
+            average_latency_ms: average_latency_ms
+          }
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/runner.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      # Sequential test-only evaluator. Concurrency belongs to the subsystem
+      # under test, not to the evaluation helper itself.
+      class Runner
+        def initialize(scorer: Scorer::ExactMatch.new)
+          @scorer = scorer
+        end
+
+        def run(dataset, callable)
+          dataset.to_a.map { |eval_case| run_one(eval_case, callable) }
+        end
+
+        private
+
+        def run_one(eval_case, callable)
+          started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+          result = callable.call(eval_case.input)
+          latency_ms = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond) - started_at
+          actual, usage = extract(result)
+          score, score_error = score_safely(
+            @scorer,
+            actual: actual,
+            expected: eval_case.expected,
+            input: eval_case.input
+          )
+          EvalResult.new(
+            eval_case: eval_case,
+            actual: actual,
+            score: score,
+            usage: usage,
+            latency_ms: latency_ms,
+            error: score_error
+          )
+        end
+
+        def extract(result)
+          result.is_a?(Hash) ? [result[:output].to_s, result[:usage]] : [result.to_s, nil]
+        end
+
+        def score_safely(scorer, **kwargs)
+          [scorer.score(**kwargs), nil]
+        rescue => error
+          [0.0, error]
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/scorer/base.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      module Scorer
+        class Base
+          def score(actual:, expected:, input: nil)
+            raise NotImplementedError, "#{self.class}#score is not implemented"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/scorer/exact_match.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      module Scorer
+        class ExactMatch < Base
+          def initialize(case_sensitive: true)
+            @case_sensitive = case_sensitive
+          end
+
+          def score(actual:, expected:, input: nil)
+            actual_value = actual.to_s.strip
+            expected_value = expected.to_s.strip
+            unless @case_sensitive
+              actual_value = actual_value.downcase
+              expected_value = expected_value.downcase
+            end
+            (actual_value == expected_value) ? 1.0 : 0.0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/scorer/includes_scorer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      module Scorer
+        class IncludesScorer < Base
+          def initialize(case_sensitive: false)
+            @case_sensitive = case_sensitive
+          end
+
+          def score(actual:, expected:, input: nil)
+            actual_value = actual.to_s
+            expected_value = expected.to_s
+            unless @case_sensitive
+              actual_value = actual_value.downcase
+              expected_value = expected_value.downcase
+            end
+            actual_value.include?(expected_value) ? 1.0 : 0.0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/scorer/llm_judge.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      module Scorer
+        class LlmJudge < Base
+          DEFAULT_PROMPT = <<~PROMPT
+            You are an impartial judge evaluating the quality of an AI assistant response.
+            Rate the response on a scale from 0.0 (completely wrong or unhelpful) to 1.0 (perfect).
+            Respond with ONLY a single decimal number between 0.0 and 1.0 — no other text.
+
+            Question: %<input>s
+            Expected answer: %<expected>s
+            Actual response: %<actual>s
+
+            Score:
+          PROMPT
+
+          def initialize(model:, prompt_template: DEFAULT_PROMPT, raise_on_error: false)
+            @model = model
+            @prompt_template = prompt_template
+            @raise_on_error = raise_on_error
+          end
+
+          def score(actual:, expected:, input: nil)
+            prompt = format(
+              @prompt_template,
+              input: input.to_s,
+              expected: expected.to_s,
+              actual: actual.to_s
+            )
+            response = Phronomy::Runtime.instance.offload.submit do
+              RubyLLM.chat(model: @model).ask(prompt)
+            end.blocking_wait
+            response.content.to_s.strip.scan(/-?\d+\.?\d*/).first.to_f.clamp(0.0, 1.0)
+          rescue => error
+            raise if @raise_on_error
+            warn "[LlmJudge] Scoring failed: #{error.message}"
+            0.0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval/scorer.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    module Eval
+      module Scorer
+      end
+    end
+  end
+end

data/lib/phronomy/testing/eval.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Testing
+    # Test-only evaluation helpers.
+    module Eval
+    end
+  end
+end

data/lib/phronomy/testing/fake_clock.rb

@@ -2,32 +2,16 @@
 
 module Phronomy
   module Testing
-    # A deterministic, manually-advanced clock for use in tests.
-    #
-    # Replaces real +Process.clock_gettime+ calls so that time-sensitive code
-    # can be tested without relying on wall-clock sleeps.
-    #
-    # @example
-    #   clock = Phronomy::Testing::FakeClock.new
-    #   clock.now        # => 0.0
-    #   clock.advance(5) # advance by 5 seconds
-    #   clock.now        # => 5.0
+    # Deterministic manually-advanced clock for tests.
     class FakeClock
-      # @return [Float] the current logical time in seconds since the epoch (t=0)
       attr_reader :now
 
       def initialize
         @now = 0.0
-        @callbacks = [] # [[fire_at, block], ...]
+        @callbacks = []
         @mutex = Mutex.new
       end
 
-      # Advance the clock by +seconds+ and fire any registered callbacks whose
-      # deadline has passed.
-      #
-      # @param seconds [Numeric]
-      # @return [self]
-      # @api private
       def advance(seconds)
         @mutex.synchronize do
           @now += seconds.to_f
@@ -36,65 +20,34 @@ module Phronomy
         self
       end
 
-      # Register a one-shot callback that fires when the clock reaches +at+.
-      #
-      # @param at [Numeric] logical time to fire
-      # @yield called with no arguments when the clock reaches +at+
-      # @return [self]
-      # @api private
       def at(at, &block)
         @mutex.synchronize { @callbacks << [at.to_f, block] }
         self
       end
 
-      # Schedule a one-shot callback to fire after +seconds+ from the current
-      # logical time.  This is the same interface as {Runtime::TimerQueue#schedule}
-      # so that a +FakeClock+ can be passed as a +timer_queue:+ argument in tests.
-      #
-      # @param seconds [Numeric] delay in logical seconds
-      # @yield called when the clock reaches the scheduled time
-      # @return [self]
-      # @api private
       def schedule(seconds:, &block)
         at(@now + seconds.to_f, &block)
       end
 
-      # Returns the number of pending (un-fired) callbacks.
-      # @return [Integer]
-      # @api private
       def pending_callbacks
         @mutex.synchronize { @callbacks.size }
       end
 
-      # Returns the logical time of the next pending callback, or +nil+ if
-      # there are no pending callbacks.
-      #
-      # @return [Float, nil]
-      # @api private
       def next_timer_at
         @mutex.synchronize { @callbacks.min_by { |(t, _)| t }&.first }
       end
 
-      # Advance the clock exactly to the next pending callback and fire it.
-      # Raises +RuntimeError+ when there are no pending callbacks.
-      #
-      # @return [self]
-      # @api private
       def advance_to_next_timer
         target = next_timer_at
         raise "No pending timers to advance to" unless target
-
         advance(target - @now)
       end
 
-      # Returns descriptive entries for all pending callbacks.
-      # Used by {Phronomy::Runtime::FakeScheduler#pending_timers}.
-      #
-      # @return [Array<Hash>] each entry: +{ fire_at:, description: nil }+
-      # @api private
       def pending_timer_entries
         @mutex.synchronize do
-          @callbacks.sort_by { |(t, _)| t }.map { |(t, _)| {fire_at: t, description: nil} }
+          @callbacks.sort_by { |(t, _)| t }.map do |(time, _)|
+            {fire_at: time, description: nil}
+          end
         end
       end
 
@@ -102,7 +55,7 @@ module Phronomy
 
       def fire_expired_callbacks!
         fired, @callbacks = @callbacks.partition { |(t, _)| t <= @now }
-        fired.sort_by { |(t, _)| t }.each { |(_, cb)| cb.call }
+        fired.sort_by { |(t, _)| t }.each { |(_, callback)| callback.call }
       end
     end
   end

data/lib/phronomy/testing.rb

@@ -1,12 +1,8 @@
 # frozen_string_literal: true
 
 module Phronomy
-  # Test helpers for deterministic, timer-independent testing.
-  #
-  # @example
-  #   require "phronomy/testing"
-  #   clock = Phronomy::Testing::FakeClock.new
-  #   scheduler = Phronomy::Testing::FakeScheduler.new
+  # Test-only helpers. Nothing under this namespace participates in the
+  # production Runtime control plane.
   module Testing
   end
 end