rspec-agents 0.1.0

data/lib/rspec/agents/serialization.rb

@@ -0,0 +1,374 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require "securerandom"
+require_relative "metadata"
+
+# Event Serialization System
+#
+# Provides data structures for capturing and persisting test run data.
+# See 2026_01_27_event_serialization-design.md for architecture details.
+
+module RSpec
+  module Agents
+    module Serialization
+      ALPINE_VERSION = "3.14.8"
+
+      # Shared serialization helpers
+      module SerializationHelpers
+        def serialize_value(value)
+          case value
+          when nil then nil
+          when Time then value.iso8601(3)
+          when Array then value.map { |v| serialize_value(v) }
+          when Hash then value.transform_values { |v| serialize_value(v) }
+          when ->(v) { v.respond_to?(:to_h) && !v.is_a?(Hash) } then value.to_h
+          else value
+          end
+        end
+
+        def parse_time(value)
+          return nil unless value
+          value.is_a?(Time) ? value : Time.parse(value)
+        end
+
+        def get(hash, key)
+          return hash[key] if hash.key?(key)
+          hash[key.to_s]
+        end
+
+        def get_array(hash, key)
+          get(hash, key) || []
+        end
+
+        def wrap_metadata(value)
+          value.is_a?(Metadata) ? value : Metadata.new(value || {})
+        end
+      end
+
+      # For Data.define classes - provides to_h with serialization
+      module DataClassMethods
+        include SerializationHelpers
+
+        def to_h
+          members.each_with_object({}) do |key, hash|
+            hash[key] = serialize_value(send(key))
+          end
+        end
+      end
+
+      # Class methods for from_h on Data.define classes
+      module DataClassFromH
+        include SerializationHelpers
+      end
+
+      # =========================================================================
+      # Immutable Data Classes
+      # =========================================================================
+
+      ExceptionData = Data.define(:class_name, :message, :backtrace) do
+        include DataClassMethods
+        extend DataClassFromH
+
+        def initialize(class_name:, message:, backtrace: [])
+          super(class_name: class_name, message: message, backtrace: Array(backtrace).first(10))
+        end
+
+        def self.from_h(hash)
+          return nil unless hash
+          new(class_name: get(hash, :class_name), message: get(hash, :message), backtrace: get_array(hash, :backtrace))
+        end
+      end
+
+      MessageData = Data.define(:role, :content, :timestamp, :source, :metadata) do
+        include DataClassMethods
+        extend DataClassFromH
+
+        def initialize(role:, content:, timestamp:, source: nil, metadata: {})
+          super(role: role.to_s, content: content, timestamp: timestamp, source: source&.to_s, metadata: wrap_metadata(metadata))
+        end
+
+        def self.from_h(hash)
+          return nil unless hash
+          new(role: get(hash, :role), content: get(hash, :content), timestamp: parse_time(get(hash, :timestamp)),
+              source: get(hash, :source), metadata: get(hash, :metadata) || {})
+        end
+      end
+
+      ToolCallData = Data.define(:name, :arguments, :result, :error, :timestamp, :metadata) do
+        include DataClassMethods
+        extend DataClassFromH
+
+        def initialize(name:, arguments:, timestamp:, result: nil, error: nil, metadata: {})
+          super(name: name.to_s, arguments: arguments || {}, result: result, error: error,
+                timestamp: timestamp, metadata: wrap_metadata(metadata))
+        end
+
+        def self.from_h(hash)
+          return nil unless hash
+          new(name: get(hash, :name), arguments: get(hash, :arguments) || {}, result: get(hash, :result),
+              error: get(hash, :error), timestamp: parse_time(get(hash, :timestamp)), metadata: get(hash, :metadata) || {})
+        end
+      end
+
+      EvaluationData = Data.define(:name, :description, :passed, :reasoning, :timestamp, :mode, :type, :failure_message, :metadata) do
+        include DataClassMethods
+        extend DataClassFromH
+
+        def initialize(name:, description:, passed:, timestamp:, reasoning: nil, mode: nil, type: nil, failure_message: nil, metadata: {})
+          super(name: name, description: description, passed: passed, reasoning: reasoning,
+                timestamp: timestamp, mode: mode&.to_sym, type: type&.to_sym, failure_message: failure_message,
+                metadata: wrap_metadata(metadata))
+        end
+
+        # Whether this is a soft evaluation (quality metric, doesn't affect test pass/fail)
+        def soft?
+          mode == :soft
+        end
+
+        # Whether this is a hard expectation (affects test pass/fail)
+        def hard?
+          mode == :hard
+        end
+
+        def self.from_h(hash)
+          return nil unless hash
+          new(name: get(hash, :name), description: get(hash, :description), passed: get(hash, :passed),
+              reasoning: get(hash, :reasoning), timestamp: parse_time(get(hash, :timestamp)),
+              mode: get(hash, :mode)&.to_sym, type: get(hash, :type)&.to_sym,
+              failure_message: get(hash, :failure_message), metadata: get(hash, :metadata) || {})
+        end
+      end
+
+      # Scenario data for serialization
+      # Captures the scenario definition used in a test
+      ScenarioData = Data.define(:id, :name, :goal, :personality, :context, :verification, :data) do
+        include DataClassMethods
+        extend DataClassFromH
+
+        def initialize(id:, name:, goal:, personality: nil, context: nil, verification: nil, data: {})
+          super(id: id, name: name, goal: goal, personality: personality,
+                context: context, verification: verification, data: data || {})
+        end
+
+        def self.from_h(hash)
+          return nil unless hash
+          new(
+            id:           get(hash, :id),
+            name:         get(hash, :name),
+            goal:         get(hash, :goal),
+            personality:  get(hash, :personality),
+            context:      get(hash, :context),
+            verification: get(hash, :verification),
+            data:         get(hash, :data) || {}
+          )
+        end
+
+        # Create from a Scenario object
+        def self.from_scenario(scenario)
+          return nil unless scenario
+          new(
+            id:           scenario.identifier,
+            name:         scenario[:name] || scenario[:id],
+            goal:         scenario[:goal],
+            personality:  scenario[:personality],
+            context:      scenario[:context],
+            verification: scenario[:verification],
+            data:         scenario.to_h
+          )
+        end
+      end
+
+      SummaryStats = Data.define(:example_count, :passed_count, :failed_count, :pending_count, :total_duration_ms) do
+        include DataClassMethods
+      end
+
+      # =========================================================================
+      # Mutable Data Classes (built incrementally during test execution)
+      # =========================================================================
+
+      class TurnData
+        include SerializationHelpers
+        extend DataClassFromH
+
+        attr_reader :number, :user_message, :metadata
+        attr_accessor :agent_response, :tool_calls
+
+        def initialize(number:, user_message:, agent_response: nil, tool_calls: [], topic: nil, metadata: {})
+          @number = number
+          @user_message = user_message
+          @agent_response = agent_response
+          @tool_calls = tool_calls || []
+          @topic = topic&.to_s
+          @metadata = wrap_metadata(metadata)
+        end
+
+        def topic=(value)
+          @topic = value&.to_s
+        end
+
+        attr_reader :topic
+
+        def to_h
+          { number: @number, user_message: @user_message&.to_h, agent_response: @agent_response&.to_h,
+            tool_calls: @tool_calls.map(&:to_h), topic: @topic, metadata: @metadata.to_h }
+        end
+
+        def self.from_h(hash)
+          return nil unless hash
+          new(number: get(hash, :number), user_message: MessageData.from_h(get(hash, :user_message)),
+              agent_response: MessageData.from_h(get(hash, :agent_response)),
+              tool_calls: get_array(hash, :tool_calls).map { |tc| ToolCallData.from_h(tc) },
+              topic: get(hash, :topic), metadata: get(hash, :metadata) || {})
+        end
+      end
+
+      class ConversationData
+        include SerializationHelpers
+        extend DataClassFromH
+
+        attr_reader :started_at, :turns, :metadata
+        attr_accessor :ended_at, :final_topic
+
+        def initialize(started_at:, ended_at: nil, turns: [], final_topic: nil, metadata: {})
+          @started_at = started_at
+          @ended_at = ended_at
+          @turns = turns || []
+          @final_topic = final_topic&.to_s
+          @metadata = wrap_metadata(metadata)
+        end
+
+        def add_turn(turn) = @turns << turn
+        def current_turn = @turns.last
+
+        def to_h
+          { started_at: serialize_value(@started_at), ended_at: serialize_value(@ended_at),
+            turns: @turns.map(&:to_h), final_topic: @final_topic, metadata: @metadata.to_h }
+        end
+
+        def self.from_h(hash)
+          return nil unless hash
+          new(started_at: parse_time(get(hash, :started_at)), ended_at: parse_time(get(hash, :ended_at)),
+              turns: get_array(hash, :turns).map { |t| TurnData.from_h(t) },
+              final_topic: get(hash, :final_topic), metadata: get(hash, :metadata) || {})
+        end
+      end
+
+      class ExampleData
+        include SerializationHelpers
+        extend DataClassFromH
+
+        attr_reader :id, :stable_id, :canonical_path, :file, :description, :location, :started_at, :evaluations, :metadata
+        attr_accessor :status, :finished_at, :duration_ms, :exception, :conversation, :scenario_id
+
+        def initialize(id:, file:, description:, location:, started_at:, status: :pending,
+                       stable_id: nil, canonical_path: nil, scenario_id: nil,
+                       finished_at: nil, duration_ms: nil, exception: nil, conversation: nil, evaluations: [], metadata: {})
+          @id, @file, @description, @location = id, file, description, location
+          @stable_id, @canonical_path = stable_id, canonical_path
+          @scenario_id = scenario_id
+          @status, @started_at, @finished_at, @duration_ms = status.to_sym, started_at, finished_at, duration_ms
+          @exception, @conversation, @evaluations = exception, conversation, evaluations || []
+          @metadata = wrap_metadata(metadata)
+        end
+
+        def add_evaluation(evaluation) = @evaluations << evaluation
+
+        def to_h
+          { id: @id, stable_id: @stable_id, canonical_path: @canonical_path, scenario_id: @scenario_id,
+            file: @file, description: @description, location: @location, status: @status.to_s,
+            started_at: serialize_value(@started_at), finished_at: serialize_value(@finished_at),
+            duration_ms: @duration_ms, exception: @exception&.to_h, conversation: @conversation&.to_h,
+            evaluations: @evaluations.map(&:to_h), metadata: @metadata.to_h }
+        end
+
+        def self.from_h(hash)
+          return nil unless hash
+          new(id: get(hash, :id), stable_id: get(hash, :stable_id), canonical_path: get(hash, :canonical_path),
+              scenario_id: get(hash, :scenario_id),
+              file: get(hash, :file), description: get(hash, :description),
+              location: get(hash, :location), status: get(hash, :status)&.to_sym || :pending,
+              started_at: parse_time(get(hash, :started_at)), finished_at: parse_time(get(hash, :finished_at)),
+              duration_ms: get(hash, :duration_ms), exception: ExceptionData.from_h(get(hash, :exception)),
+              conversation: ConversationData.from_h(get(hash, :conversation)),
+              evaluations: get_array(hash, :evaluations).map { |e| EvaluationData.from_h(e) },
+              metadata: get(hash, :metadata) || {})
+        end
+      end
+
+      class RunData
+        include SerializationHelpers
+        extend DataClassFromH
+
+        attr_reader :run_id, :started_at, :seed, :examples, :scenarios
+        attr_accessor :finished_at
+
+        def initialize(run_id:, started_at:, finished_at: nil, seed: nil, examples: {}, scenarios: {})
+          @run_id, @started_at, @finished_at, @seed = run_id, started_at, finished_at, seed
+          @examples = examples || {}
+          @scenarios = scenarios || {}
+        end
+
+        def add_example(example_data) = @examples[example_data.id] = example_data
+        def example(id) = @examples[id]
+
+        # Register a scenario in the scenarios hash
+        # @param scenario_data [ScenarioData] The scenario to register
+        # @return [String] The scenario ID
+        def register_scenario(scenario_data)
+          return nil unless scenario_data
+          @scenarios[scenario_data.id] = scenario_data unless @scenarios.key?(scenario_data.id)
+          scenario_data.id
+        end
+
+        # Get a scenario by ID
+        # @param id [String] The scenario ID
+        # @return [ScenarioData, nil]
+        def scenario(id)
+          @scenarios[id]
+        end
+
+        def summary
+          counts = @examples.each_value.with_object(passed: 0, failed: 0, pending: 0, duration: 0) do |ex, c|
+            c[ex.status] += 1 if c.key?(ex.status)
+            c[:duration] += ex.duration_ms.to_i
+          end
+          SummaryStats.new(example_count: @examples.size, passed_count: counts[:passed],
+                           failed_count: counts[:failed], pending_count: counts[:pending],
+                           total_duration_ms: counts[:duration])
+        end
+
+        def to_h
+          { run_id: @run_id, started_at: serialize_value(@started_at), finished_at: serialize_value(@finished_at),
+            seed: @seed, scenarios: @scenarios.transform_values(&:to_h), examples: @examples.transform_values(&:to_h) }
+        end
+
+        def self.from_h(hash)
+          return nil unless hash
+          new(run_id: get(hash, :run_id), started_at: parse_time(get(hash, :started_at)),
+              finished_at: parse_time(get(hash, :finished_at)), seed: get(hash, :seed),
+              scenarios: (get(hash, :scenarios) || {}).transform_values { |s| ScenarioData.from_h(s) },
+              examples: (get(hash, :examples) || {}).transform_values { |e| ExampleData.from_h(e) })
+        end
+      end
+
+      # =========================================================================
+      # JsonFile - Simple JSON file read/write utility
+      # =========================================================================
+
+      class JsonFile
+        class << self
+          def write(path, run_data)
+            FileUtils.mkdir_p(File.dirname(path))
+            File.write(path, JSON.pretty_generate(run_data.to_h))
+          end
+
+          def read(path)
+            RunData.from_h(JSON.parse(File.read(path)))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/agents/simulator_config.rb

@@ -0,0 +1,336 @@
+module RSpec
+  module Agents
+    # Configuration for the user simulator
+    # Supports inheritance through RSpec nesting with specific merge behaviors
+    #
+    # Inheritance rules (per design doc Section 3.4):
+    # - String replaces block, block replaces string
+    # - Block + block = notes merged
+    # - Rules are always accumulated
+    # - max_turns/stop_when replaced by child
+    # - goal/template are test-level only (not inheritable)
+    class SimulatorConfig
+      attr_reader :role_value, :personality_value, :context_value
+      attr_reader :rules, :max_turns, :stop_when, :goal, :template
+      attr_reader :topic_overrides
+
+      def initialize
+        @role_value = nil
+        @role_type = nil  # :string or :block
+
+        @personality_value = nil
+        @personality_type = nil
+        @personality_notes = []
+
+        @context_value = nil
+        @context_type = nil
+        @context_notes = []
+
+        @rules = []
+        @max_turns = nil
+        @stop_when = nil
+        @goal = nil
+        @template = nil
+        @topic_overrides = {}
+
+        @current_notes_target = nil  # Tracks which block we're in
+      end
+
+      # DSL: Set the user's role
+      # @param text [String, nil] Role description (or nil to use block)
+      # @yield Block for complex role definition (not implemented yet)
+      def role(text = nil, &block)
+        if block_given?
+          @role_type = :block
+          @role_value = block
+        else
+          @role_type = :string
+          @role_value = text
+        end
+      end
+
+      # DSL: Set the user's personality
+      # @param text [String, nil] Personality description (or nil to use block)
+      # @yield Block for personality with notes
+      def personality(text = nil, &block)
+        if block_given?
+          @personality_type = :block
+          @personality_value = nil
+          @current_notes_target = :personality
+          instance_eval(&block)
+          @current_notes_target = nil
+        else
+          @personality_type = :string
+          @personality_value = text
+          @personality_notes = []
+        end
+      end
+
+      # DSL: Set the context
+      # @param text [String, nil] Context description (or nil to use block)
+      # @yield Block for context with notes
+      def context(text = nil, &block)
+        if block_given?
+          @context_type = :block
+          @context_value = nil
+          @current_notes_target = :context
+          instance_eval(&block)
+          @current_notes_target = nil
+        else
+          @context_type = :string
+          @context_value = text
+          @context_notes = []
+        end
+      end
+
+      # DSL: Add a note (used within personality/context blocks)
+      # @param text [String] Note text
+      def note(text)
+        case @current_notes_target
+        when :personality
+          @personality_notes << text
+        when :context
+          @context_notes << text
+        else
+          # Default to personality notes if called outside a block
+          @personality_notes << text
+        end
+      end
+
+      # DSL: Add a rule
+      # @param type [Symbol, Proc] :should, :should_not, or a lambda for dynamic rules
+      # @param text [String, nil] Rule description (for :should/:should_not)
+      # @yield Optional block for dynamic rules
+      def rule(type_or_lambda = nil, text = nil, &block)
+        if block_given?
+          @rules << { type: :dynamic, block: block }
+        elsif type_or_lambda.is_a?(Proc)
+          @rules << { type: :dynamic, block: type_or_lambda }
+        elsif type_or_lambda.is_a?(Symbol) && text
+          @rules << { type: type_or_lambda, text: text }
+        end
+      end
+
+      # DSL: Set maximum turns
+      # @param count [Integer]
+      attr_writer :max_turns
+
+      # For DSL compatibility
+      def max_turns(count = nil)
+        if count.nil?
+          @max_turns
+        else
+          @max_turns = count
+        end
+      end
+
+      # DSL: Set stop condition
+      # @yield [turn, conversation] Block returning true to stop
+      def stop_when(&block)
+        if block_given?
+          @stop_when = block
+        else
+          @stop_when
+        end
+      end
+
+      # DSL: Set the goal (test-level only)
+      # @param text [String]
+      def goal(text = nil)
+        if text.nil?
+          @goal
+        else
+          @goal = text
+        end
+      end
+
+      # DSL: Set the template path (test-level only)
+      # @param path [String, nil] Path to ERB template file
+      def template(path = nil)
+        if path.nil?
+          @template
+        else
+          @template = path.to_s
+        end
+      end
+
+      # DSL: Override settings during a specific topic
+      # @param topic_name [Symbol] Topic name
+      # @yield Block with overrides
+      def during_topic(topic_name, &block)
+        override = self.class.new
+        override.instance_eval(&block)
+        @topic_overrides[topic_name.to_sym] = override
+      end
+
+      # Get the effective role as an array
+      # @return [Array<String>] Role items (empty array if not set)
+      def effective_role
+        case @role_type
+        when :string
+          [@role_value]
+        when :block
+          result = @role_value&.call
+          result ? [result] : []
+        else
+          []
+        end
+      end
+
+      # Get the effective personality as an array
+      # @return [Array<String>] Personality items (empty array if not set)
+      def effective_personality
+        case @personality_type
+        when :string
+          [@personality_value]
+        when :block
+          @personality_notes.dup
+        else
+          []
+        end
+      end
+
+      # Get the effective context as an array
+      # @return [Array<String>] Context items (empty array if not set)
+      def effective_context
+        case @context_type
+        when :string
+          [@context_value]
+        when :block
+          @context_notes.dup
+        else
+          []
+        end
+      end
+
+      # Merge with a child config using inheritance rules
+      # @param child [SimulatorConfig] Child configuration
+      # @return [SimulatorConfig] New merged configuration
+      def merge(child)
+        result = self.class.new
+
+        # Role: String/block replacement rules
+        merge_setting(result, :role, child)
+
+        # Personality: String/block replacement, block+block = merge notes
+        merge_personality(result, child)
+
+        # Context: String/block replacement, block+block = merge notes
+        merge_context(result, child)
+
+        # Rules: Always accumulated
+        result.instance_variable_set(:@rules, @rules + child.rules)
+
+        # max_turns: Child replaces parent
+        result.instance_variable_set(:@max_turns, child.max_turns || @max_turns)
+
+        # stop_when: Child replaces parent
+        result.instance_variable_set(:@stop_when, child.instance_variable_get(:@stop_when) || @stop_when)
+
+        # goal: Test-level only, child takes precedence
+        result.instance_variable_set(:@goal, child.instance_variable_get(:@goal) || @goal)
+
+        # template: Test-level only, child takes precedence
+        result.instance_variable_set(:@template, child.template || @template)
+
+        # Topic overrides: Merge hashes
+        merged_overrides = @topic_overrides.merge(child.topic_overrides)
+        result.instance_variable_set(:@topic_overrides, merged_overrides)
+
+        result
+      end
+
+      # Get config with topic-specific overrides applied
+      # @param topic_name [Symbol] Current topic
+      # @return [SimulatorConfig]
+      def for_topic(topic_name)
+        override = @topic_overrides[topic_name.to_sym]
+        override ? merge(override) : self
+      end
+
+      def to_h
+        {
+          role:        effective_role,
+          personality: effective_personality,
+          context:     effective_context,
+          rules:       rules,
+          max_turns:   @max_turns,
+          goal:        @goal
+        }
+      end
+
+      private
+
+      def merge_setting(result, name, child)
+        parent_type = instance_variable_get(:"@#{name}_type")
+        parent_value = instance_variable_get(:"@#{name}_value")
+        child_type = child.instance_variable_get(:"@#{name}_type")
+        child_value = child.instance_variable_get(:"@#{name}_value")
+
+        if child_type
+          # Child has a value - it replaces parent
+          result.instance_variable_set(:"@#{name}_type", child_type)
+          result.instance_variable_set(:"@#{name}_value", child_value)
+        else
+          # No child value - inherit from parent
+          result.instance_variable_set(:"@#{name}_type", parent_type)
+          result.instance_variable_set(:"@#{name}_value", parent_value)
+        end
+      end
+
+      def merge_personality(result, child)
+        parent_type = @personality_type
+        child_type = child.instance_variable_get(:@personality_type)
+
+        if child_type == :string
+          # String replaces anything
+          result.instance_variable_set(:@personality_type, :string)
+          result.instance_variable_set(:@personality_value, child.personality_value)
+          result.instance_variable_set(:@personality_notes, [])
+        elsif child_type == :block && parent_type == :block
+          # Block + Block = merge notes
+          result.instance_variable_set(:@personality_type, :block)
+          result.instance_variable_set(:@personality_value, nil)
+          result.instance_variable_set(:@personality_notes, @personality_notes + child.instance_variable_get(:@personality_notes))
+        elsif child_type == :block
+          # Block replaces string
+          result.instance_variable_set(:@personality_type, :block)
+          result.instance_variable_set(:@personality_value, nil)
+          result.instance_variable_set(:@personality_notes, child.instance_variable_get(:@personality_notes))
+        else
+          # Inherit from parent
+          result.instance_variable_set(:@personality_type, parent_type)
+          result.instance_variable_set(:@personality_value, @personality_value)
+          result.instance_variable_set(:@personality_notes, @personality_notes.dup)
+        end
+      end
+
+      def merge_context(result, child)
+        parent_type = @context_type
+        child_type = child.instance_variable_get(:@context_type)
+
+        if child_type == :string
+          # String replaces anything
+          result.instance_variable_set(:@context_type, :string)
+          result.instance_variable_set(:@context_value, child.context_value)
+          result.instance_variable_set(:@context_notes, [])
+        elsif child_type == :block && parent_type == :block
+          # Block + Block = merge notes
+          result.instance_variable_set(:@context_type, :block)
+          result.instance_variable_set(:@context_value, nil)
+          result.instance_variable_set(:@context_notes, @context_notes + child.instance_variable_get(:@context_notes))
+        elsif child_type == :block
+          # Block replaces string
+          result.instance_variable_set(:@context_type, :block)
+          result.instance_variable_set(:@context_value, nil)
+          result.instance_variable_set(:@context_notes, child.instance_variable_get(:@context_notes))
+        else
+          # Inherit from parent
+          result.instance_variable_set(:@context_type, parent_type)
+          result.instance_variable_set(:@context_value, @context_value)
+          result.instance_variable_set(:@context_notes, @context_notes.dup)
+        end
+      end
+    end
+  end
+end