rspec-llm 0.1.0

data/lib/rspec/llm/adapters/fake.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module RSpec
+  module LLM
+    module Adapters
+      # In-memory programmable adapter. Use in unit-style specs to avoid
+      # hitting a real LLM while still exercising matcher and DSL behavior.
+      #
+      #   fake = RSpec::LLM::Adapters::Fake.new
+      #   fake.respond_to("Summarize: foo").with("foo summary")
+      #   fake.respond_to_pattern(/^Summarize/).with { |prompt| "summary of #{prompt}" }
+      #   fake.default("I don't know")
+      #   fake.embed_with { |text| [text.length.to_f, 0.0, 0.0] }
+      class Fake < Base
+        class UnstubbedPromptError < StandardError; end
+
+        # Builder returned by #respond_to / #respond_to_pattern so callers can
+        # chain `.with(text)` or `.with { |prompt| ... }`.
+        class Stub
+          def initialize(parent, matcher)
+            @parent = parent
+            @matcher = matcher
+          end
+
+          def with(text = nil, &block)
+            raise ArgumentError, "pass text or a block" if text.nil? && block.nil?
+
+            @parent.send(:register, @matcher, text || block)
+            @parent
+          end
+        end
+
+        def initialize(client = nil)
+          super
+          @stubs = []
+          @default = nil
+          @embedder = nil
+          @call_log = []
+        end
+
+        attr_reader :call_log
+
+        def respond_to(prompt)
+          Stub.new(self, prompt)
+        end
+
+        def respond_to_pattern(regex)
+          Stub.new(self, regex)
+        end
+
+        def default(text = nil, &block)
+          raise ArgumentError, "pass text or a block" if text.nil? && block.nil?
+
+          @default = text || block
+          self
+        end
+
+        def embed_with(&block)
+          @embedder = block
+          self
+        end
+
+        def chat(messages)
+          prompt = last_user_message(normalize_messages(messages))
+          @call_log << prompt
+          response = lookup(prompt)
+          response.is_a?(Proc) ? response.call(prompt) : response
+        end
+
+        def embed(text)
+          raise NotImplementedError, "configure with #embed_with { |text| vector }" unless @embedder
+
+          @embedder.call(text)
+        end
+
+        def reset!
+          @stubs.clear
+          @default = nil
+          @embedder = nil
+          @call_log.clear
+          self
+        end
+
+        private
+
+        def register(matcher, value)
+          @stubs << [matcher, value]
+        end
+
+        def last_user_message(messages)
+          last_user = messages.reverse.find { |m| m[:role].to_s == "user" }
+          (last_user || messages.last)[:content].to_s
+        end
+
+        def lookup(prompt)
+          stub = @stubs.find { |matcher, _| matches?(matcher, prompt) }
+          return stub.last if stub
+          return @default if @default
+
+          raise UnstubbedPromptError,
+                "No fake response stubbed for prompt: #{prompt.inspect}. " \
+                "Use fake.respond_to(...).with(...) or fake.default(...)."
+        end
+
+        def matches?(matcher, prompt)
+          case matcher
+          when Regexp then matcher.match?(prompt)
+          else matcher == prompt
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/llm/adapters/langchain.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module RSpec
+  module LLM
+    module Adapters
+      # Adapter for the langchainrb gem (https://github.com/patterns-ai-core/langchainrb).
+      # Wraps a Langchain::LLM::* instance.
+      class Langchain < Base
+        def chat(messages)
+          normalized = normalize_messages(messages)
+          response = client.chat(messages: normalized)
+          extract_content(response)
+        end
+
+        def embed(text)
+          response = client.embed(text: text)
+          extract_vector(response)
+        end
+
+        private
+
+        def extract_content(response)
+          return response if response.is_a?(String)
+          return response.chat_completion if response.respond_to?(:chat_completion) && response.chat_completion
+          return response.completion if response.respond_to?(:completion) && response.completion
+
+          response.to_s
+        end
+
+        def extract_vector(response)
+          return response if response.is_a?(Array)
+          return response.embedding if response.respond_to?(:embedding) && response.embedding
+          return response.embeddings.first if response.respond_to?(:embeddings) && response.embeddings
+
+          raise NotImplementedError, "Cannot extract vector from #{response.class}"
+        end
+      end
+    end
+  end
+end

data/lib/rspec/llm/adapters/ruby_llm.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module RSpec
+  module LLM
+    module Adapters
+      # Adapter for the ruby_llm gem (https://github.com/crmne/ruby_llm).
+      # Wraps a RubyLLM::Chat instance and exposes the common adapter surface.
+      class RubyLLM < Base
+        def chat(messages)
+          normalized = normalize_messages(messages)
+          last = normalized.last
+          system_msgs = normalized[0..-2].select { |m| m[:role] == "system" }
+          if system_msgs.any? && client.respond_to?(:with_instructions)
+            system_msgs.each do |m|
+              client.with_instructions(m[:content])
+            end
+          end
+
+          response = client.ask(last[:content])
+          extract_content(response)
+        end
+
+        def embed(text)
+          return @embedder.call(text) if @embedder
+
+          raise NotImplementedError, "RubyLLM.embed is not available" unless defined?(::RubyLLM) && ::RubyLLM.respond_to?(:embed)
+
+          result = ::RubyLLM.embed(text)
+          vectors = result.respond_to?(:vectors) ? result.vectors : result
+          vectors.is_a?(Array) && vectors.first.is_a?(Array) ? vectors.first : vectors
+        end
+
+        # Override the embedder (useful in tests). Accepts a callable.
+        def with_embedder(callable)
+          @embedder = callable
+          self
+        end
+
+        private
+
+        def extract_content(response)
+          return response if response.is_a?(String)
+          return response.content if response.respond_to?(:content)
+
+          response.to_s
+        end
+      end
+    end
+  end
+end

data/lib/rspec/llm/configuration.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module RSpec
+  module LLM
+    # Holds gem-wide configuration: the default client used by specs, the judge
+    # model used by LLM-as-judge matchers, an embedder callable used by the
+    # similarity matcher, and tunable thresholds/prompts.
+    class Configuration
+      DEFAULT_SIMILARITY_THRESHOLD = 0.8
+
+      DEFAULT_JUDGE_PROMPT = <<~PROMPT
+        You are a strict evaluator. Read the response and decide whether it satisfies the criterion.
+        Reply with YES or NO on the first line, then a single short sentence explaining your decision.
+
+        Response:
+        %<response>s
+
+        Criterion:
+        %<criterion>s
+      PROMPT
+
+      attr_accessor :client, :judge, :embedder, :similarity_threshold, :judge_prompt_template
+
+      def initialize
+        @similarity_threshold = DEFAULT_SIMILARITY_THRESHOLD
+        @judge_prompt_template = DEFAULT_JUDGE_PROMPT
+      end
+
+      def client_adapter
+        return nil if client.nil?
+
+        Adapters::Base.wrap(client)
+      end
+
+      def judge_adapter
+        return nil if judge.nil?
+
+        Adapters::Base.wrap(judge)
+      end
+    end
+  end
+end

data/lib/rspec/llm/dsl.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module RSpec
+  module LLM
+    # Lightweight DSL for grouping LLM evaluations. Sugar over `describe` + `it`.
+    #
+    #   RSpec.describe_llm "Summarizer" do
+    #     evaluate "single sentence",
+    #       prompt: "Summarize: ...",
+    #       expect: [pass_llm_judge("is one sentence"), match_llm_intent("a summary")]
+    #   end
+    module DSL
+      # Class-level helper exposed inside `describe_llm` blocks.
+      module GroupMethods
+        def evaluate(name, prompt:, expect:, client: nil)
+          it(name) do
+            adapter = client ? RSpec::LLM::Adapters::Base.wrap(client) : RSpec::LLM.client
+            raise RSpec::LLM::Error, "No LLM client configured" if adapter.nil?
+
+            response = adapter.chat(prompt)
+            Array(expect).each do |matcher|
+              ::RSpec::Expectations::PositiveExpectationHandler.handle_matcher(response, matcher)
+            end
+          end
+        end
+      end
+
+      # Defines RSpec.describe_llm — a sugar for RSpec.describe that extends the
+      # resulting example group with GroupMethods so `evaluate` is available
+      # inside the user's block.
+      def describe_llm(*args, &user_block)
+        describe(*args) do
+          extend(::RSpec::LLM::DSL::GroupMethods)
+          class_exec(&user_block) if user_block
+        end
+      end
+    end
+  end
+end

data/lib/rspec/llm/helpers.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module RSpec
+  module LLM
+    # Methods mixed into example groups via RSpec.configure (see rspec.rb).
+    module Helpers
+      # The configured client wrapped as an Adapter. Returns nil if not configured.
+      def llm
+        RSpec::LLM.client
+      end
+
+      # Configure a Fake adapter and yield it for stubbing. Replaces the global
+      # client for the duration of the example. After the example RSpec::LLM is
+      # reset via the `after` hook installed in rspec.rb.
+      def stub_llm
+        fake = RSpec::LLM::Adapters::Fake.new
+        yield fake if block_given?
+        RSpec::LLM.configuration.client = fake
+        fake
+      end
+
+      # Same, but installs the fake as the judge.
+      def stub_llm_judge
+        fake = RSpec::LLM::Adapters::Fake.new
+        yield fake if block_given?
+        RSpec::LLM.configuration.judge = fake
+        fake
+      end
+    end
+  end
+end

data/lib/rspec/llm/matchers/be_semantically_similar_to.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module RSpec
+  module LLM
+    module Matchers
+      # Embeds the actual and expected texts via the configured embedder and
+      # asserts cosine similarity >= threshold (default from configuration,
+      # overridable with .within(0.9)).
+      class BeSemanticallySimilarTo
+        def initialize(expected)
+          @expected = expected
+          @threshold = nil
+        end
+
+        def within(threshold)
+          @threshold = threshold
+          self
+        end
+
+        def matches?(actual)
+          @actual = actual.to_s
+          @actual_vec = embedder.call(@actual)
+          @expected_vec = embedder.call(@expected.to_s)
+          @similarity = cosine(@actual_vec, @expected_vec)
+          @similarity >= threshold_value
+        end
+
+        def description
+          "be semantically similar to #{@expected.inspect} (>= #{threshold_value})"
+        end
+
+        def failure_message
+          "expected response to be semantically similar to expected text " \
+            "(similarity #{format_similarity} < threshold #{threshold_value}).\n" \
+            "Expected: #{@expected.inspect}\nActual:   #{@actual.inspect}"
+        end
+
+        def failure_message_when_negated
+          "expected response NOT to be semantically similar to expected text " \
+            "(similarity #{format_similarity} >= threshold #{threshold_value}).\n" \
+            "Expected: #{@expected.inspect}\nActual:   #{@actual.inspect}"
+        end
+
+        private
+
+        def embedder
+          RSpec::LLM.configuration.embedder or raise(
+            RSpec::LLM::Error,
+            "No embedder configured. Set RSpec::LLM.configure { |c| c.embedder = ->(text) { ... } }."
+          )
+        end
+
+        def threshold_value
+          @threshold || RSpec::LLM.configuration.similarity_threshold
+        end
+
+        def cosine(vec_a, vec_b)
+          raise ArgumentError, "embedder returned empty vector" if vec_a.empty? || vec_b.empty?
+          raise ArgumentError, "vector dimensions differ (#{vec_a.size} vs #{vec_b.size})" if vec_a.size != vec_b.size
+
+          dot = vec_a.zip(vec_b).sum { |a, b| a * b }
+          mag_a = Math.sqrt(vec_a.sum { |v| v * v })
+          mag_b = Math.sqrt(vec_b.sum { |v| v * v })
+          return 0.0 if mag_a.zero? || mag_b.zero?
+
+          dot / (mag_a * mag_b)
+        end
+
+        def format_similarity
+          format("%.4f", @similarity)
+        end
+      end
+    end
+  end
+end

data/lib/rspec/llm/matchers/match_json_schema.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "json"
+require "json-schema"
+
+module RSpec
+  module LLM
+    module Matchers
+      # Asserts the actual value parses as JSON and conforms to the provided
+      # JSON Schema (a Hash, JSON string, or schema file path).
+      class MatchJsonSchema
+        def initialize(schema)
+          @schema = schema
+        end
+
+        def matches?(actual)
+          @actual = actual
+          @parsed = parse(actual)
+          return false if @parse_error
+
+          @errors = JSON::Validator.fully_validate(@schema, @parsed)
+          @errors.empty?
+        end
+
+        def description
+          "match JSON schema"
+        end
+
+        def failure_message
+          if @parse_error
+            "expected response to parse as JSON, but got: #{@parse_error}\n\nResponse:\n#{@actual}"
+          else
+            "expected response to match JSON schema, but got errors:\n#{@errors.join("\n")}\n\nResponse:\n#{@actual}"
+          end
+        end
+
+        def failure_message_when_negated
+          "expected response NOT to match JSON schema, but it did.\n\nResponse:\n#{@actual}"
+        end
+
+        private
+
+        def parse(actual)
+          return actual if actual.is_a?(Hash) || actual.is_a?(Array)
+
+          JSON.parse(actual.to_s)
+        rescue JSON::ParserError => e
+          @parse_error = e.message
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/rspec/llm/matchers/match_llm_intent.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module RSpec
+  module LLM
+    module Matchers
+      # Variant of PassLlmJudge that frames the criterion as an intent the
+      # response should satisfy. Sugar — the heavy lifting is identical.
+      class MatchLlmIntent < PassLlmJudge
+        def initialize(intent)
+          super("The response matches the following intent: #{intent}")
+          @intent = intent
+        end
+
+        def description
+          "match LLM intent: #{@intent.inspect}"
+        end
+
+        def failure_message
+          "expected response to match intent #{@intent.inspect}, " \
+            "but judge said: #{format_reason}\n\nResponse:\n#{@actual}"
+        end
+
+        def failure_message_when_negated
+          "expected response NOT to match intent #{@intent.inspect}, " \
+            "but judge said: #{format_reason}\n\nResponse:\n#{@actual}"
+        end
+      end
+    end
+  end
+end

data/lib/rspec/llm/matchers/pass_llm_judge.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module RSpec
+  module LLM
+    module Matchers
+      # LLM-as-judge matcher. Asks the configured judge model whether the
+      # actual response satisfies the given criterion. Parses YES/NO from the
+      # first non-whitespace token of the judge's reply.
+      class PassLlmJudge
+        def initialize(criterion)
+          @criterion = criterion
+          @judge = nil
+        end
+
+        # Override the judge for this matcher invocation (optional).
+        def using(judge)
+          @judge = judge
+          self
+        end
+
+        def matches?(actual)
+          @actual = actual.to_s
+          @verdict_text = judge_adapter.chat(prompt_for(@actual, @criterion))
+          @verdict, @reason = parse_verdict(@verdict_text)
+          @verdict == true
+        end
+
+        def description
+          "pass LLM judge with criterion: #{@criterion.inspect}"
+        end
+
+        def failure_message
+          "expected response to pass judge criterion #{@criterion.inspect}, " \
+            "but judge said: #{format_reason}\n\nResponse:\n#{@actual}"
+        end
+
+        def failure_message_when_negated
+          "expected response NOT to pass judge criterion #{@criterion.inspect}, " \
+            "but judge said: #{format_reason}\n\nResponse:\n#{@actual}"
+        end
+
+        private
+
+        def judge_adapter
+          @judge || RSpec::LLM.judge or raise(
+            RSpec::LLM::Error,
+            "No judge configured. Call RSpec::LLM.configure { |c| c.judge = ... } " \
+            "or .using(client) on the matcher."
+          )
+        end
+
+        def prompt_for(response, criterion)
+          format(RSpec::LLM.configuration.judge_prompt_template, response: response, criterion: criterion)
+        end
+
+        def parse_verdict(text)
+          stripped = text.to_s.strip
+          first_token = stripped.split(/\s+/, 2).first.to_s.upcase
+          verdict = first_token.start_with?("YES")
+          reason = stripped.lines.drop(1).join.strip
+          reason = stripped if reason.empty?
+          [verdict, reason]
+        end
+
+        def format_reason
+          @reason.empty? ? @verdict_text.to_s.strip : @reason
+        end
+      end
+    end
+  end
+end

data/lib/rspec/llm/matchers.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative "matchers/pass_llm_judge"
+require_relative "matchers/match_llm_intent"
+require_relative "matchers/match_json_schema"
+require_relative "matchers/be_semantically_similar_to"
+
+module RSpec
+  module LLM
+    # Module to be included in example groups (RSpec.configure does this
+    # automatically via lib/rspec/llm/rspec.rb). Exposes the matcher DSL.
+    module Matchers
+      def pass_llm_judge(criterion)
+        RSpec::LLM::Matchers::PassLlmJudge.new(criterion)
+      end
+
+      def match_llm_intent(intent)
+        RSpec::LLM::Matchers::MatchLlmIntent.new(intent)
+      end
+
+      def match_json_schema(schema)
+        RSpec::LLM::Matchers::MatchJsonSchema.new(schema)
+      end
+
+      def be_semantically_similar_to(expected)
+        RSpec::LLM::Matchers::BeSemanticallySimilarTo.new(expected)
+      end
+    end
+  end
+end

data/lib/rspec/llm/rspec.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+# Wire matchers, helpers, and DSL into RSpec automatically when the gem is
+# required. Users only need `require "rspec/llm"` in their spec_helper.
+
+RSpec.configure do |config|
+  config.include RSpec::LLM::Matchers
+  config.include RSpec::LLM::Helpers
+end
+
+# Expose `describe_llm` at the top level by extending RSpec itself.
+RSpec.extend(RSpec::LLM::DSL)

data/lib/rspec/llm/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module RSpec
+  module LLM
+    VERSION = "0.1.0"
+  end
+end

data/lib/rspec/llm.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "rspec/core"
+require "rspec/expectations"
+
+require_relative "llm/version"
+require_relative "llm/configuration"
+require_relative "llm/adapters/base"
+require_relative "llm/adapters/fake"
+require_relative "llm/adapters/ruby_llm"
+require_relative "llm/adapters/langchain"
+require_relative "llm/matchers"
+require_relative "llm/helpers"
+require_relative "llm/dsl"
+require_relative "llm/rspec"
+
+module RSpec
+  module LLM
+    class Error < StandardError; end
+
+    class << self
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      def configure
+        yield configuration
+      end
+
+      def reset!
+        @configuration = Configuration.new
+      end
+
+      def client
+        configuration.client_adapter
+      end
+
+      def judge
+        configuration.judge_adapter || client
+      end
+    end
+  end
+end

data/sig/rspec/llm.rbs

@@ -0,0 +1,6 @@
+module Rspec
+  module Llm
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end