rspec-agents 0.1.0

data/lib/rspec/agents/topic.rb

@@ -0,0 +1,307 @@
+module RSpec
+  module Agents
+    # Wraps a single expectation with its type, mode, and data
+    class TopicExpectation
+      attr_reader :type, :mode, :data
+
+      def initialize(type, mode, data)
+        @type = type  # :quality, :match, :no_match, :grounding, :tool_call, :forbidden_tool, :forbidden_claims, :custom
+        @mode = mode  # :soft or :hard
+        @data = data  # Type-specific data hash
+      end
+
+      def soft?
+        @mode == :soft
+      end
+
+      def hard?
+        @mode == :hard
+      end
+    end
+
+    # Represents a distinct phase or state in a conversation
+    # Each topic has characteristics, triggers, invariants, and an intent
+    class Topic
+      attr_reader :name, :triggers, :invariants
+      attr_accessor :successors  # Set by TopicGraph
+
+      def initialize(name, &block)
+        @name = name.to_sym
+        @characteristic_text = nil
+        @agent_intent_text = nil
+        @triggers = Triggers.new
+        @invariants = InvariantSet.new
+        @successors = []
+
+        instance_eval(&block) if block_given?
+      end
+
+      # DSL: Set the topic's characteristic description
+      # Used by LLM judge to identify when conversation is in this topic
+      def characteristic(text)
+        @characteristic_text = text.strip
+      end
+
+      # DSL: Set the agent's intended behavior during this topic
+      def agent_intent(text)
+        @agent_intent_text = text.strip
+      end
+
+      # DSL: Define triggers for deterministic topic classification
+      def triggers(&block)
+        if block_given?
+          @triggers = Triggers.new(&block)
+        else
+          @triggers
+        end
+      end
+
+      # DSL: Expect agent to satisfy criteria (hard)
+      # @param to_satisfy [Array<Symbol>] Criterion names
+      # @param to_match [Regexp] Pattern agent response must match
+      # @param not_to_match [Regexp] Pattern agent response must NOT match
+      def expect_agent(to_satisfy: nil, to_match: nil, not_to_match: nil)
+        @invariants.add_expectation(:quality, :hard, { criteria: to_satisfy }) if to_satisfy
+        @invariants.add_expectation(:match, :hard, { pattern: to_match }) if to_match
+        @invariants.add_expectation(:no_match, :hard, { pattern: not_to_match }) if not_to_match
+      end
+
+      # DSL: Expect agent claims to be grounded in tool results (hard)
+      # @param claim_types [Array<Symbol>] Types of claims (:venues, :pricing, etc.)
+      # @param from_tools [Array<Symbol>] Tool names that should provide grounding
+      def expect_grounding(*claim_types, from_tools: [])
+        @invariants.add_expectation(:grounding, :hard, { claim_types: claim_types, from_tools: from_tools })
+      end
+
+      # DSL: Forbid agent from making ungrounded claims (hard)
+      # @param claim_types [Array<Symbol>] Types of claims to forbid
+      def forbid_claims(*claim_types)
+        @invariants.add_expectation(:forbidden_claims, :hard, { claim_types: claim_types })
+      end
+
+      # DSL: Expect agent to call a specific tool (hard)
+      # @param tool_name [Symbol] Tool name
+      def expect_tool_call(tool_name)
+        @invariants.add_expectation(:tool_call, :hard, { tool_name: tool_name })
+      end
+
+      # DSL: Forbid agent from calling a specific tool (hard)
+      # @param tool_name [Symbol] Tool name
+      def forbid_tool_call(tool_name)
+        @invariants.add_expectation(:forbidden_tool, :hard, { tool_name: tool_name })
+      end
+
+      # DSL: Custom expectation with block (hard)
+      # @param description [String] Description of expectation
+      # @yield [turn, conversation] Block that returns true/false
+      def expect(description, &block)
+        @invariants.add_expectation(:custom, :hard, { description: description, block: block })
+      end
+
+      # DSL: Soft evaluation - quality criteria
+      # @param to_satisfy [Array<Symbol>] Criterion names
+      # @param to_match [Regexp] Pattern agent response must match
+      # @param not_to_match [Regexp] Pattern agent response must NOT match
+      def evaluate_agent(to_satisfy: nil, to_match: nil, not_to_match: nil)
+        @invariants.add_expectation(:quality, :soft, { criteria: to_satisfy }) if to_satisfy
+        @invariants.add_expectation(:match, :soft, { pattern: to_match }) if to_match
+        @invariants.add_expectation(:no_match, :soft, { pattern: not_to_match }) if not_to_match
+      end
+
+      # DSL: Soft evaluation - grounding
+      # @param claim_types [Array<Symbol>] Types of claims (:venues, :pricing, etc.)
+      # @param from_tools [Array<Symbol>] Tool names that should provide grounding
+      def evaluate_grounding(*claim_types, from_tools: [])
+        @invariants.add_expectation(:grounding, :soft, { claim_types: claim_types, from_tools: from_tools })
+      end
+
+      # DSL: Soft evaluation - tool calls
+      # @param tool_name [Symbol] Tool name
+      def evaluate_tool_call(tool_name)
+        @invariants.add_expectation(:tool_call, :soft, { tool_name: tool_name })
+      end
+
+      # DSL: Soft evaluation - custom
+      # @param description [String] Description of expectation
+      # @yield [turn, conversation] Block that returns true/false
+      def evaluate(description, &block)
+        @invariants.add_expectation(:custom, :soft, { description: description, block: block })
+      end
+
+      attr_reader :characteristic_text
+
+      attr_reader :agent_intent_text
+
+      def trigger_matches?(turn, conversation)
+        @triggers.any_match?(turn, conversation)
+      end
+
+      # Evaluate all invariants for turns that occurred in this topic
+      # @param turns [Array] Turns that occurred in this topic
+      # @param conversation [Object] Full conversation context
+      # @param judge [Object] LLM judge for evaluating quality/grounding
+      # @return [InvariantResults]
+      def evaluate_invariants(turns, conversation, judge)
+        @invariants.evaluate(turns, conversation, judge, @name)
+      end
+
+      def to_h
+        {
+          name:           @name,
+          characteristic: @characteristic_text,
+          agent_intent:   @agent_intent_text,
+          successors:     @successors
+        }
+      end
+    end
+
+    # Collects and evaluates invariants for a topic
+    class InvariantSet
+      def initialize
+        @expectations = []  # Array of TopicExpectation objects
+      end
+
+      # Generic method to add any type of expectation
+      def add_expectation(type, mode, data)
+        @expectations << TopicExpectation.new(type, mode, data)
+      end
+
+      def empty?
+        @expectations.empty?
+      end
+
+      # Evaluate all invariants against the given turns
+      # @return [InvariantResults]
+      def evaluate(turns, conversation, judge, topic_name)
+        results = InvariantResults.new(topic_name)
+
+        @expectations.each do |expectation|
+          case expectation.type
+          when :quality
+            # LLM-based evaluation - mark as pending
+            results.add_pending(:quality, expectation.data[:criteria], mode: expectation.mode)
+
+          when :match
+            # Pattern matching - deterministic
+            pattern = expectation.data[:pattern]
+            matched = turns.any? { |t| pattern.match?(t.agent_response&.text.to_s) }
+            results.add(
+              :match,
+              pattern.inspect,
+              matched,
+              matched ? nil : "No response matched #{pattern.inspect}",
+              mode: expectation.mode
+            )
+
+          when :no_match
+            # Forbidden pattern - deterministic
+            pattern = expectation.data[:pattern]
+            violated = turns.any? { |t| pattern.match?(t.agent_response&.text.to_s) }
+            results.add(
+              :no_match,
+              pattern.inspect,
+              !violated,
+              violated ? "Response matched forbidden pattern #{pattern.inspect}" : nil,
+              mode: expectation.mode
+            )
+
+          when :tool_call
+            # Tool call expectation - deterministic
+            all_tool_calls = turns.flat_map { |t| t.agent_response&.tool_calls || [] }
+            tool_name = expectation.data[:tool_name]
+            called = all_tool_calls.any? { |tc| tc.name == tool_name }
+            results.add(
+              :tool_call,
+              tool_name,
+              called,
+              called ? nil : "Expected tool call to #{tool_name} but it was not called",
+              mode: expectation.mode
+            )
+
+          when :forbidden_tool
+            # Forbidden tool - deterministic
+            all_tool_calls = turns.flat_map { |t| t.agent_response&.tool_calls || [] }
+            tool_name = expectation.data[:tool_name]
+            called = all_tool_calls.any? { |tc| tc.name == tool_name }
+            results.add(
+              :forbidden_tool,
+              tool_name,
+              !called,
+              called ? "Forbidden tool #{tool_name} was called" : nil,
+              mode: expectation.mode
+            )
+
+          when :grounding
+            # LLM-based evaluation - mark as pending
+            results.add_pending(:grounding, expectation.data, mode: expectation.mode)
+
+          when :forbidden_claims
+            # LLM-based evaluation - mark as pending
+            results.add_pending(:forbidden_claims, expectation.data[:claim_types], mode: expectation.mode)
+
+          when :custom
+            # Code-based evaluation - deterministic
+            passed = turns.all? { |turn| expectation.data[:block].call(turn, conversation) }
+            results.add(
+              :custom,
+              expectation.data[:description],
+              passed,
+              passed ? nil : "Custom expectation failed: #{expectation.data[:description]}",
+              mode: expectation.mode
+            )
+          end
+        end
+
+        results
+      end
+    end
+
+    # Stores results of invariant evaluation
+    class InvariantResults
+      attr_reader :topic_name, :results, :pending
+
+      def initialize(topic_name)
+        @topic_name = topic_name
+        @results = []
+        @pending = []  # Requires judge evaluation
+      end
+
+      def add(type, description, passed, failure_message, mode: :hard)
+        @results << {
+          type:            type,
+          description:     description,
+          passed:          passed,
+          failure_message: failure_message,
+          mode:            mode
+        }
+      end
+
+      def add_pending(type, data, mode: :hard)
+        @pending << { type: type, data: data, mode: mode }
+      end
+
+      def passed?
+        # Only check hard expectations
+        @results.select { |r| r[:mode] == :hard }.all? { |r| r[:passed] } &&
+          @pending.select { |p| p[:mode] == :hard }.empty?
+      end
+
+      def failures
+        # Only return hard failures
+        @results.reject { |r| r[:passed] || r[:mode] == :soft }
+      end
+
+      def soft_failures
+        @results.select { |r| !r[:passed] && r[:mode] == :soft }
+      end
+
+      def hard_failures
+        @results.select { |r| !r[:passed] && r[:mode] == :hard }
+      end
+
+      def has_pending?
+        !@pending.empty?
+      end
+    end
+  end
+end

data/lib/rspec/agents/topic_graph.rb

@@ -0,0 +1,236 @@
+module RSpec
+  module Agents
+    # Topic graph validation errors
+    # Defined at module level so they can be caught before RSpec wraps them
+    class TopicGraphValidationError < StandardError; end
+    class DuplicateTopicError < TopicGraphValidationError; end
+    class UndefinedTopicError < TopicGraphValidationError; end
+    class SelfLoopError < TopicGraphValidationError; end
+    class UnreachableTopicError < TopicGraphValidationError; end
+
+    # Directed graph of topics representing conversation flow
+    # Validates structure and provides traversal methods
+    class TopicGraph
+      attr_reader :initial_topic, :topics
+
+      def initialize
+        @topics = {}
+        @edges = {}  # topic_name => [successor_names]
+        @initial_topic = nil
+        @topic_order = []  # Tracks order for initial topic detection
+      end
+
+      # Add a topic with its successors
+      # @param topic [Topic] Topic instance
+      # @param next_topics [Symbol, Array<Symbol>, nil] Successor topic name(s)
+      def add_topic(topic, next_topics: nil)
+        name = topic.name
+
+        raise DuplicateTopicError, "Topic :#{name} is already defined" if @topics.key?(name)
+
+        @topics[name] = topic
+        @edges[name] = normalize_next(next_topics)
+        @topic_order << name
+        @initial_topic ||= name
+      end
+
+      # Reference a shared topic and wire it into the graph
+      # @param name [Symbol] Topic name (must exist in shared topics or already added)
+      # @param next_topics [Symbol, Array<Symbol>, nil] Successor topic name(s)
+      # @param shared_topics [Hash] Hash of shared topic definitions
+      def use_topic(name, next_topics: nil, shared_topics: {})
+        name = name.to_sym
+
+        if @topics.key?(name)
+          # Topic already added, just update edges
+          @edges[name] = normalize_next(next_topics)
+        elsif shared_topics.key?(name)
+          # Copy from shared topics
+          shared_topic = shared_topics[name]
+          @topics[name] = shared_topic
+          @edges[name] = normalize_next(next_topics)
+          @topic_order << name
+          @initial_topic ||= name
+        else
+          # Create placeholder - will be validated later
+          @topics[name] = nil
+          @edges[name] = normalize_next(next_topics)
+          @topic_order << name
+          @initial_topic ||= name
+        end
+      end
+
+      # Validate the graph structure
+      # @raise [ValidationError] If validation fails
+      def validate!
+        validate_referential_integrity!
+        validate_no_self_loops!
+        validate_connectivity!
+        wire_successors!
+      end
+
+      # Get successor topic names for a topic
+      # @param topic_name [Symbol] Topic name
+      # @return [Array<Symbol>]
+      def successors_of(topic_name)
+        @edges[topic_name.to_sym] || []
+      end
+
+      # Check if target is reachable from start
+      # @param start [Symbol] Starting topic name
+      # @param target [Symbol] Target topic name
+      # @return [Boolean]
+      def reachable_from?(start, target)
+        visited = Set.new
+        queue = [start.to_sym]
+
+        while queue.any?
+          current = queue.shift
+          return true if current == target.to_sym
+
+          next if visited.include?(current)
+          visited << current
+
+          queue.concat(successors_of(current))
+        end
+
+        false
+      end
+
+      # Get a topic by name
+      # @param name [Symbol] Topic name
+      # @return [Topic, nil]
+      def [](name)
+        @topics[name.to_sym]
+      end
+
+      def topic_names
+        @topics.keys
+      end
+
+      def empty?
+        @topics.empty?
+      end
+
+      def size
+        @topics.size
+      end
+
+      # Get all terminal topics (no successors)
+      def terminal_topics
+        @topics.keys.select { |name| successors_of(name).empty? }
+      end
+
+      # Classify which topic a turn belongs to, given the current topic
+      # Uses conservative classification: current topic triggers checked first,
+      # then successor triggers, then LLM fallback if judge provided.
+      #
+      # @param turn [Turn] The turn to classify
+      # @param conversation [Conversation] Full conversation context
+      # @param current_topic [Symbol] The current topic name
+      # @param judge [Judge, nil] Optional judge for LLM-based classification
+      # @return [Symbol] The classified topic name
+      def classify(turn, conversation, current_topic, judge: nil)
+        current_topic = current_topic.to_sym
+        current_topic_obj = @topics[current_topic]
+
+        # 1. Check current topic's triggers first (conservative stay)
+        if current_topic_obj&.trigger_matches?(turn, conversation)
+          return current_topic
+        end
+
+        # 2. Check successor topics' triggers
+        successors = successors_of(current_topic)
+        successors.each do |successor_name|
+          successor = @topics[successor_name]
+          if successor&.trigger_matches?(turn, conversation)
+            return successor_name
+          end
+        end
+
+        # 3. No triggers matched - use LLM classification if judge available and successors exist
+        if judge && successors.any?
+          possible_topics = ([current_topic_obj] + successors.map { |s| @topics[s] }).compact
+          return judge.classify_topic(turn, conversation, possible_topics)
+        end
+
+        # 4. No successors and no triggers (or no judge) - stay in current topic
+        current_topic
+      end
+
+      def to_h
+        {
+          initial_topic: @initial_topic,
+          topics:        @topics.transform_values { |t| t&.to_h },
+          edges:         @edges
+        }
+      end
+
+      private
+
+      def normalize_next(next_topics)
+        case next_topics
+        when nil
+          []
+        when Symbol
+          [next_topics]
+        when Array
+          next_topics.map(&:to_sym)
+        else
+          raise ArgumentError, "next: must be a Symbol, Array of Symbols, or nil"
+        end
+      end
+
+      def validate_referential_integrity!
+        @edges.each do |source, targets|
+          targets.each do |target|
+            unless @topics.key?(target)
+              raise UndefinedTopicError, "Topic :#{source} references undefined topic :#{target}"
+            end
+          end
+        end
+
+        # Check for placeholder topics (used but never defined)
+        @topics.each do |name, topic|
+          if topic.nil?
+            raise UndefinedTopicError, "Topic :#{name} was referenced but never defined"
+          end
+        end
+      end
+
+      def validate_no_self_loops!
+        @edges.each do |source, targets|
+          if targets.include?(source)
+            raise SelfLoopError, "Topic :#{source} has a self-loop (next: [:#{source}]). Self-loops are not allowed; use conservative topic classification instead."
+          end
+        end
+      end
+
+      def validate_connectivity!
+        return if @topics.empty?
+
+        reachable = Set.new
+        queue = [@initial_topic]
+
+        while queue.any?
+          current = queue.shift
+          next if reachable.include?(current)
+
+          reachable << current
+          queue.concat(successors_of(current))
+        end
+
+        unreachable = @topics.keys - reachable.to_a
+        if unreachable.any?
+          raise UnreachableTopicError, "Topics #{unreachable.map { |t| ":#{t}" }.join(', ')} are not reachable from initial topic :#{@initial_topic}"
+        end
+      end
+
+      def wire_successors!
+        @topics.each do |name, topic|
+          topic.successors = successors_of(name)
+        end
+      end
+    end
+  end
+end

data/lib/rspec/agents/triggers.rb

@@ -0,0 +1,122 @@
+module RSpec
+  module Agents
+    # Enables deterministic topic classification without LLM calls
+    # When a trigger matches, the topic is classified without invoking the judge
+    class Triggers
+      def initialize(&block)
+        @triggers = []
+        instance_eval(&block) if block_given?
+      end
+
+      # Trigger when agent calls a specific tool
+      # @param name [Symbol] Tool name
+      # @param with_params [Hash] Optional parameter constraints (values can be Regexp or exact match)
+      def on_tool_call(name, with_params: nil)
+        @triggers << ToolCallTrigger.new(name, with_params)
+      end
+
+      # Trigger when agent response matches a pattern
+      # @param pattern [Regexp] Pattern to match against agent response text
+      def on_response_match(pattern)
+        @triggers << ResponseMatchTrigger.new(pattern)
+      end
+
+      # Trigger when user message matches a pattern
+      # @param pattern [Regexp] Pattern to match against user message text
+      def on_user_match(pattern)
+        @triggers << UserMatchTrigger.new(pattern)
+      end
+
+      # Trigger after N turns in a specific topic
+      # @param topic [Symbol] Topic name
+      # @param count [Integer] Number of turns
+      def after_turns_in(topic, count:)
+        @triggers << TurnsInTopicTrigger.new(topic, count)
+      end
+
+      # Trigger on custom condition
+      # @param condition [Proc] Lambda receiving (turn, conversation)
+      def on_condition(condition)
+        @triggers << ConditionTrigger.new(condition)
+      end
+
+      # Check if any trigger matches the current turn
+      # @param turn [Object] Current turn (has agent_response, user_message)
+      # @param conversation [Object] Full conversation context
+      # @return [Boolean]
+      def any_match?(turn, conversation)
+        @triggers.any? { |trigger| trigger.match?(turn, conversation) }
+      end
+
+      def empty?
+        @triggers.empty?
+      end
+
+      def count
+        @triggers.count
+      end
+
+      # Individual trigger types
+
+      class ToolCallTrigger
+        def initialize(name, params)
+          @name = name.to_sym
+          @params = params
+        end
+
+        def match?(turn, _conversation)
+          return false unless turn.respond_to?(:agent_response) && turn.agent_response
+
+          turn.agent_response.has_tool_call?(@name, params: @params)
+        end
+      end
+
+      class ResponseMatchTrigger
+        def initialize(pattern)
+          @pattern = pattern
+        end
+
+        def match?(turn, _conversation)
+          return false unless turn.respond_to?(:agent_response) && turn.agent_response
+
+          @pattern.match?(turn.agent_response.text.to_s)
+        end
+      end
+
+      class UserMatchTrigger
+        def initialize(pattern)
+          @pattern = pattern
+        end
+
+        def match?(turn, _conversation)
+          return false unless turn.respond_to?(:user_message)
+
+          @pattern.match?(turn.user_message.to_s)
+        end
+      end
+
+      class TurnsInTopicTrigger
+        def initialize(topic, count)
+          @topic = topic.to_sym
+          @count = count
+        end
+
+        def match?(_turn, conversation)
+          return false unless conversation.respond_to?(:turns_in_topic)
+
+          conversation.turns_in_topic(@topic) >= @count
+        end
+      end
+
+      class ConditionTrigger
+        def initialize(condition)
+          @condition = condition
+        end
+
+        def match?(turn, conversation)
+          @condition.call(turn, conversation)
+        end
+      end
+    end
+  end
+end