rspec-agents 0.1.0

data/lib/rspec/agents/doc/scenario_guide.md

@@ -0,0 +1,289 @@
+# RSpec Agents Scenario Guide
+
+This guide provides practical examples for writing agent tests using the rspec-agents DSL.
+
+## Quick Reference
+
+### Criteria Types
+
+```ruby
+# 1. Named criterion (defined at describe/context level)
+criterion :friendly, "The agent's response should be friendly"
+expect(agent).to satisfy(:friendly)
+
+# 2. Adhoc criterion (inline string description, evaluated by LLM)
+expect(agent).to satisfy("Der Agent fragt nach Details oder zeigt das Formular an")
+
+# 3. Lambda criterion (code-based, no LLM)
+expect(agent).to satisfy(->(turn) { turn.agent_response.text.length <= 500 })
+
+# 4. Named lambda criterion
+expect(agent).to satisfy(:concise, ->(turn) { turn.agent_response.text.length <= 300 })
+```
+
+### Soft vs Hard Assertions
+
+```ruby
+# Soft: records result, continues even if fails
+evaluate(agent).to satisfy(:friendly)
+
+# Hard: test fails immediately if not met
+expect(agent).to satisfy(:friendly)
+```
+
+---
+
+## Common Patterns
+
+### Basic Scripted Conversation
+
+```ruby
+RSpec.describe "Room Booking Agent", type: :agent do
+  criterion :friendly, "The agent's response should be friendly"
+
+  it "handles booking request" do
+    user.says "Hi, I need to book a meeting room"
+
+    evaluate(agent).to satisfy(:friendly)
+    expect(agent).not_to call_tool(:book_room)
+
+    user.says "The Blue Room for tomorrow at 2pm"
+
+    expect(agent).to call_tool(:check_availability)
+  end
+end
+```
+
+### Using Adhoc Criterions
+
+Adhoc criterions are useful when you need a one-off evaluation without defining a named criterion. The string description is passed directly to the LLM judge.
+
+```ruby
+it "handles form display" do
+  user.says "I want to book a room"
+
+  # German description - LLM evaluates naturally
+  expect(agent).to satisfy("Der Agent fragt nach Details oder zeigt das Formular an")
+
+  # English description
+  evaluate(agent).to satisfy("The agent acknowledges the request")
+
+  # Mix named and adhoc
+  evaluate(agent).to satisfy(:friendly)
+  expect(agent).to satisfy("Response contains actionable next steps")
+end
+```
+
+### Combining Criterion Types
+
+```ruby
+it "validates response quality" do
+  user.says "Search for venues in Stuttgart"
+
+  # Named criterion
+  evaluate(agent).to satisfy(:friendly)
+
+  # Adhoc criterion for specific requirement
+  expect(agent).to satisfy("The agent confirms the search location")
+
+  # Lambda for deterministic checks
+  expect(agent).to satisfy(->(turn) {
+    turn.agent_response.tool_calls.any? { |tc| tc.name == :search_venues }
+  })
+
+  # Multiple in one call
+  evaluate(agent).to satisfy(
+    :helpful,
+    "Response is professional",
+    ->(turn) { turn.agent_response.text.length < 1000 }
+  )
+end
+```
+
+### Scenario-Based Testing (from JSON file)
+
+```ruby
+RSpec.describe "Event Booking Agent", type: :agent do
+  criterion :friendly, "The agent's response should be friendly"
+  criterion :helpful, "The agent should move toward the user's goal"
+
+  scenario_set "venue_searches", from: "scenarios/venue_search.json" do |scenario|
+    it "handles #{scenario[:name]}" do
+      user.simulate do
+        goal scenario[:goal]
+        personality scenario[:personality]
+      end
+
+      # Adhoc criterion using scenario data
+      expect(agent).to satisfy("The agent addresses: #{scenario[:goal]}")
+    end
+  end
+end
+```
+
+### Scenario-Based Testing (inline array)
+
+Scenarios can be defined directly in the test file using `scenarios:` instead of `from:`. This is useful for self-contained tests or dynamically generated scenarios.
+
+```ruby
+RSpec.describe "Event Booking Agent", type: :agent do
+  criterion :friendly, "The agent's response should be friendly"
+  criterion :helpful, "The agent should move toward the user's goal"
+
+  scenarios = [
+    {
+      id: "weihnachtsfeier",
+      name: "Unternehmens-Weihnachtsfeier",
+      goal: "Eine festliche Weihnachtsfeier für 60 Mitarbeiter in Berlin organisieren",
+      personality: "Festlich gestimmt, achtet auf Details wie Dekoration und Menü",
+      context: ["Dezember-Termin", "Abendveranstaltung mit Dinner", "Unterhaltungsprogramm gewünscht"]
+    },
+    {
+      id: "produktlaunch",
+      name: "Produktlaunch-Event",
+      goal: "Ein Produktlaunch-Event für 100 Gäste in Hamburg organisieren",
+      personality: "Marketing-orientiert, achtet auf Präsentationstechnik und Impression",
+      context: ["Presse und Kunden eingeladen", "Moderne Location gewünscht", "Catering wichtig"]
+    },
+    {
+      id: "vertriebstagung",
+      name: "Vertriebstagung",
+      goal: "Eine zweitägige Vertriebstagung für 50 Außendienstmitarbeiter in Düsseldorf",
+      personality: "Ergebnisorientiert, fokussiert auf Motivation und Schulung",
+      context: ["Motivationstraining geplant", "Award-Verleihung am Abend", "Networking wichtig"]
+    },
+    {
+      id: "klausurtagung",
+      name: "Management-Klausurtagung",
+      goal: "Eine vertrauliche Klausurtagung für 12 Führungskräfte in einem ruhigen Hotel im Schwarzwald",
+      personality: "Diskret, bevorzugt ruhige und abgeschiedene Locations",
+      context: ["Strategische Themen", "Absolute Vertraulichkeit", "Keine Ablenkungen"]
+    },
+    {
+      id: "azubi_onboarding",
+      name: "Azubi-Onboarding",
+      goal: "Ein Onboarding-Event für 20 neue Auszubildende in der Nähe von München",
+      personality: "Jugendlich, achtet auf abwechslungsreiches Programm",
+      context: ["Junge Teilnehmer", "Teambuilding-Aktivitäten wichtig", "Lockere Atmosphäre"]
+    }
+  ]
+
+  scenario_set "corporate_events", scenarios: scenarios do |scenario|
+    it "handles #{scenario[:name]}" do
+      user.simulate do
+        goal scenario[:goal]
+        personality scenario[:personality]
+        context { scenario[:context].each { |c| note c } }
+      end
+
+      evaluate(agent).to satisfy(:friendly)
+      evaluate(agent).to satisfy(:helpful)
+    end
+  end
+end
+```
+
+**When to use each approach:**
+
+| Approach | Use When |
+|----------|----------|
+| `from: "file.json"` | Scenarios shared across multiple test files |
+| `scenarios: [...]` | Self-contained tests, dynamically generated scenarios |
+
+---
+
+## When to Use Each Criterion Type
+
+| Type | Use When | Example |
+|------|----------|---------|
+| Named | Reused across multiple tests | `:friendly`, `:helpful` |
+| Adhoc | One-off, test-specific requirement | `"Der Agent zeigt das Formular an"` |
+| Lambda | Deterministic, no LLM needed | `->(turn) { turn.text.length < 500 }` |
+| Named Lambda | Reusable code check with a name | `:concise, ->(turn) { ... }` |
+
+### Adhoc Criterion Best Practices
+
+1. **Use natural language**: Write descriptions as you would explain to a human
+2. **Be specific**: "The agent asks for the event date" is better than "Agent asks questions"
+3. **Language flexibility**: Use any language - the LLM judge handles German, English, etc.
+4. **Mix with named criteria**: Use adhoc for edge cases, named for common patterns
+
+```ruby
+# Good: specific, actionable
+expect(agent).to satisfy("The agent provides at least 3 venue options")
+expect(agent).to satisfy("Der Agent fragt nach dem Budget")
+
+# Avoid: too vague
+expect(agent).to satisfy("Good response")
+expect(agent).to satisfy("Agent works correctly")
+```
+
+---
+
+## Tool Call Assertions
+
+```ruby
+# Basic tool call check
+expect(agent).to call_tool(:search_venues)
+
+# With parameters
+expect(agent).to call_tool(:book_room).with(
+  room: "Blue Room",
+  capacity: be >= 10
+)
+
+# Negated
+expect(agent).not_to call_tool(:book_room)
+
+# Conversation-level (across all turns)
+expect(conversation).to have_tool_call(:search_venues)
+```
+
+---
+
+## Grounding Assertions
+
+```ruby
+# Verify claims are grounded in tool results
+expect(agent).to be_grounded_in(:venues, :pricing)
+
+# Specify source tools
+expect(agent).to be_grounded_in(:venues, from_tools: [:search_venues])
+
+# Forbid ungrounded claims
+expect(agent).not_to claim(:availability)
+```
+
+---
+
+## Topic Tracking
+
+**No self-loops:** A topic cannot list itself in `next:`. The tracker stays in a topic until a transition occurs naturally.
+
+```ruby
+it "progresses through booking flow" do
+  expect_conversation_to do
+    use_topic :greeting, next: :gathering_details
+    use_topic :gathering_details, next: :confirming
+    use_topic :confirming
+  end
+
+  user.says "Hello!"
+  expect(agent).to be_in_topic(:greeting)
+
+  user.says "I need a room for 10 people tomorrow"
+  expect(agent).to be_in_topic(:gathering_details)
+end
+```
+
+---
+
+## Goal Achievement
+
+```ruby
+# Check stated goal (from simulator config)
+expect(agent).to have_achieved_stated_goal
+
+# Check custom goal description
+expect(agent).to have_achieved_goal("User received venue options under budget")
+```

data/lib/rspec/agents/dsl/agent_proxy.rb

@@ -0,0 +1,141 @@
+module RSpec
+  module Agents
+    module DSL
+      # Proxy for agent state inspection in scripted tests
+      # Provides read-only access to turn executor state and assertion helpers
+      class AgentProxy
+        # @param turn_executor [TurnExecutor]
+        # @param judge [Judge, nil] Optional judge for LLM-based assertions
+        def initialize(turn_executor:, judge: nil)
+          @turn_executor = turn_executor
+          @judge         = judge
+        end
+
+        # Get the turn executor (runner)
+        # @return [TurnExecutor]
+        def runner
+          @turn_executor
+        end
+
+        # Get the conversation (delegates to turn executor)
+        # @return [Conversation]
+        def conversation
+          @turn_executor.conversation
+        end
+
+        # Get the current response object
+        # @return [AgentResponse, nil]
+        def response
+          @turn_executor.current_response
+        end
+
+        # Get the response text
+        # @return [String, nil]
+        def last_response
+          response&.text
+        end
+
+        # Get the current topic
+        # @return [Symbol, nil]
+        def current_topic
+          @turn_executor.current_topic
+        end
+
+        # Get the current turn
+        # @return [Turn, nil]
+        def current_turn
+          @turn_executor.current_turn
+        end
+
+        # Get tool calls from current response
+        # @return [Array<ToolCall>]
+        def tool_calls
+          response&.tool_calls || []
+        end
+
+        # Check if agent called a specific tool
+        # @param name [Symbol, String]
+        # @param params [Hash, nil]
+        # @return [Boolean]
+        def called_tool?(name, params: nil)
+          response&.has_tool_call?(name, params: params) || false
+        end
+
+        # Check if in a specific topic
+        # @param topic_name [Symbol]
+        # @return [Boolean]
+        def in_topic?(topic_name)
+          @turn_executor.in_topic?(topic_name)
+        end
+
+        # Check if current response matches a pattern
+        #
+        # @param pattern [Regexp] Pattern to match
+        # @return [Boolean]
+        def response_matches?(pattern)
+          response&.match?(pattern) || false
+        end
+
+        # Evaluate a criterion against the current turn
+        # Uses the Criterion class to normalize and evaluate all criterion types
+        #
+        # @param criterion [Symbol, String, Proc, Criterion] Criterion to evaluate
+        # @return [Hash] { satisfied: Boolean, reasoning: String }
+        def evaluate_criterion(criterion)
+          turn = current_turn
+          return { satisfied: false, reasoning: "No turn to evaluate" } unless turn
+
+          # Normalize to Criterion object
+          criterion_obj = criterion.is_a?(Criterion) ? criterion : Criterion.from(criterion)
+
+          criterion_obj.evaluate(
+            turn:              turn,
+            judge:             @judge,
+            conversation:      conversation,
+            criteria_registry: @judge&.criteria || {}
+          )
+        end
+
+        # Check if response is grounded in tool results
+        #
+        # @param claim_types [Array<Symbol>] Claim types to verify
+        # @param from_tools [Array<Symbol>] Tool names that should provide grounding
+        # @return [Hash] { grounded: Boolean, violations: Array }
+        def check_grounding(claim_types, from_tools: [])
+          return { grounded: true, violations: [] } unless @judge
+
+          turn = current_turn
+          return { grounded: false, violations: ["No turn to evaluate"] } unless turn
+
+          @judge.evaluate_grounding(claim_types, [turn], from_tools: from_tools)
+        end
+
+        # Check for forbidden claims
+        #
+        # @param claim_types [Array<Symbol>] Claim types that are forbidden
+        # @return [Hash] { violated: Boolean, claims_found: Array }
+        def check_forbidden_claims(claim_types)
+          return { violated: false, claims_found: [] } unless @judge
+
+          turn = current_turn
+          return { violated: false, claims_found: [] } unless turn
+
+          @judge.evaluate_forbidden_claims(claim_types, [turn])
+        end
+
+        # Check if agent demonstrates expected intent
+        #
+        # @param intent_description [String] Description of expected intent
+        # @return [Hash] { matches: Boolean, observed_intent: String, reasoning: String }
+        def check_intent(intent_description)
+          return { matches: false, reasoning: "No judge configured" } unless @judge
+
+          turn = current_turn
+          return { matches: false, reasoning: "No turn to evaluate" } unless turn
+
+          @judge.evaluate_intent(intent_description, turn)
+        end
+      end
+    end
+  end
+end

data/lib/rspec/agents/dsl/criterion_definition.rb

@@ -0,0 +1,78 @@
+module RSpec
+  module Agents
+    module DSL
+      # Criterion definition with optional examples for LLM-based evaluation
+      class CriterionDefinition
+        attr_reader :name, :good_examples, :bad_examples, :edge_cases
+        attr_reader :match_block, :match_messages_block
+
+        # @param name [Symbol, String]
+        # @param description [String, nil]
+        # @yield Optional block for complex definition
+        def initialize(name, description: nil, &block)
+          @name = name.to_sym
+          @description = description
+          @good_examples = []
+          @bad_examples = []
+          @edge_cases = []
+          @match_block = nil
+          @match_messages_block = nil
+          instance_eval(&block) if block_given?
+        end
+
+        # Get or set description
+        # @param text [String, nil]
+        # @return [String, nil]
+        def description(text = nil)
+          text.nil? ? @description : (@description = text)
+        end
+
+        # Add a good example
+        # @param text [String]
+        # @param explanation [String]
+        def good_example(text, explanation:)
+          @good_examples << { text: text, explanation: explanation }
+        end
+
+        # Add a bad example
+        # @param text [String]
+        # @param explanation [String]
+        def bad_example(text, explanation:)
+          @bad_examples << { text: text, explanation: explanation }
+        end
+
+        # Add an edge case
+        # @param text [String]
+        # @param verdict [Boolean]
+        # @param explanation [String]
+        def edge_case(text, verdict:, explanation:)
+          @edge_cases << { text: text, verdict: verdict, explanation: explanation }
+        end
+
+        # Set code-based evaluation block
+        # @yield [conversation]
+        def match(&block)
+          @match_block = block
+        end
+
+        # Set message-based evaluation block
+        # @yield [messages]
+        def match_messages(&block)
+          @match_messages_block = block
+        end
+
+        # Check if this criterion uses code-based evaluation
+        # @return [Boolean]
+        def code_based?
+          !!(@match_block || @match_messages_block)
+        end
+
+        # Check if this criterion has examples
+        # @return [Boolean]
+        def has_examples?
+          @good_examples.any? || @bad_examples.any? || @edge_cases.any?
+        end
+      end
+    end
+  end
+end

data/lib/rspec/agents/dsl/graph_builder.rb

@@ -0,0 +1,38 @@
+module RSpec
+  module Agents
+    module DSL
+      # Builder for constructing TopicGraph from DSL blocks
+      class GraphBuilder
+        # @param shared_topics [Hash<Symbol, Topic>]
+        def initialize(shared_topics = {})
+          @shared_topics = shared_topics
+          @graph = TopicGraph.new
+        end
+
+        # Wire an existing shared topic into the graph
+        # @param name [Symbol]
+        # @param next [Symbol, Array<Symbol>, nil]
+        def use_topic(name, next: nil, **options)
+          next_topics = binding.local_variable_get(:next)
+          @graph.use_topic(name.to_sym, next_topics: next_topics, shared_topics: @shared_topics)
+        end
+
+        # Define and add an inline topic to the graph
+        # @param name [Symbol]
+        # @param next [Symbol, Array<Symbol>, nil]
+        # @yield Block for topic definition
+        def topic(name, next: nil, &block)
+          next_topics = binding.local_variable_get(:next)
+          topic_instance = Topic.new(name, &block)
+          @graph.add_topic(topic_instance, next_topics: next_topics)
+        end
+
+        # Build the final TopicGraph
+        # @return [TopicGraph]
+        def build
+          @graph
+        end
+      end
+    end
+  end
+end

data/lib/rspec/agents/dsl/runner_factory.rb

@@ -0,0 +1,52 @@
+module RSpec
+  module Agents
+    module DSL
+      # Factory for creating runner instances with proper dependencies
+      # Encapsulates the complexity of wiring up runners with their dependencies
+      class RunnerFactory
+        # @param context [TestContext]
+        def initialize(context)
+          @context = context
+        end
+
+        # Build a turn executor for step-by-step test conversations
+        # @return [TurnExecutor]
+        def build_turn_executor
+          TurnExecutor.new(
+            agent:        @context.build_agent,
+            conversation: @context.conversation,
+            graph:        @context.topic_graph,
+            judge:        @context.build_judge(@context.build_llm),
+            event_bus:    @context.event_bus
+          )
+        end
+
+        # Build an agent proxy for assertions
+        # @param turn_executor [TurnExecutor]
+        # @return [AgentProxy]
+        def build_agent_proxy(turn_executor)
+          AgentProxy.new(
+            turn_executor: turn_executor,
+            judge:         @context.build_judge(@context.build_llm)
+          )
+        end
+
+        # Build a user simulator for LLM-driven conversations
+        # @param simulator_config [SimulatorConfig]
+        # @return [Runners::UserSimulator]
+        def build_user_simulator(simulator_config)
+          llm = @context.build_llm
+          Runners::UserSimulator.new(
+            agent:            @context.build_agent,
+            llm:              llm,
+            judge:            @context.build_judge(llm),
+            graph:            @context.topic_graph,
+            simulator_config: simulator_config,
+            event_bus:        @context.event_bus,
+            conversation:     @context.conversation
+          )
+        end
+      end
+    end
+  end
+end