qualspec 0.0.1

data/examples/comparison.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# Example comparison suite
+# Run with: bundle exec qualspec --html report.html examples/comparison.rb
+
+Qualspec.evaluation 'Model Comparison' do
+  candidates do
+    candidate 'gemini-flash', model: 'google/gemini-2.0-flash-001'
+    candidate 'gemini-pro', model: 'google/gemini-2.5-pro-preview'
+  end
+
+  scenario 'greeting' do
+    prompt 'Say hello in a friendly way!'
+    criterion 'responds in a friendly and welcoming manner'
+  end
+
+  scenario 'explanation' do
+    prompt 'Explain why the sky is blue in one sentence.'
+    criterion 'provides a correct scientific explanation'
+    criterion 'is concise'
+  end
+end

data/examples/model_comparison.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# Example: Comparing models for customer support quality
+#
+# Run with:
+#   QUALSPEC_API_URL=https://openrouter.ai/api/v1 \
+#   QUALSPEC_API_KEY=your_key \
+#   bundle exec qualspec examples/model_comparison.rb
+
+require 'qualspec'
+
+Qualspec.evaluation 'Customer Support Model Comparison' do
+  candidates do
+    candidate 'minimax-m2', model: 'minimax/minimax-m2'
+    candidate 'grok-flash', model: 'x-ai/grok-4.1-fast'
+    candidate 'gemini-flash', model: 'google/gemini-3-flash-preview'
+  end
+
+  # Use the built-in customer support behavior
+  behaves_like :customer_support_agent
+
+  # Add custom scenarios
+  scenario 'handles refund request' do
+    prompt "I want a refund. The product arrived broken and I'm very disappointed."
+
+    criterion 'acknowledges the problem'
+    criterion 'apologizes appropriately'
+    criterion 'offers clear next steps for the refund'
+  end
+
+  scenario 'technical troubleshooting' do
+    prompt "The app keeps crashing when I try to upload photos. I've tried restarting."
+
+    criterion 'asks clarifying questions about device/version'
+    criterion 'provides systematic troubleshooting steps'
+    criterion "doesn't assume user error"
+  end
+end

data/examples/persona_test.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+# Example: Testing if models can maintain a persona
+#
+# Run with:
+#   QUALSPEC_API_URL=https://openrouter.ai/api/v1 \
+#   QUALSPEC_API_KEY=your_key \
+#   bundle exec qualspec examples/persona_test.rb
+
+require 'qualspec'
+
+PIRATE_SYSTEM_PROMPT = <<~PROMPT
+  You are Captain Blackbeard, a friendly but dramatic pirate captain.
+  You speak with pirate slang ("arr", "matey", "ye", "shiver me timbers", etc.)
+  You're helpful but always stay in character as a pirate.
+  Never break character or mention being an AI.
+PROMPT
+
+Qualspec.evaluation 'Pirate Persona Test' do
+  candidates do
+    candidate 'gpt-4o', model: 'openai/gpt-4o', system_prompt: PIRATE_SYSTEM_PROMPT
+    candidate 'claude-sonnet', model: 'anthropic/claude-3.5-sonnet', system_prompt: PIRATE_SYSTEM_PROMPT
+  end
+
+  # Use the built-in persona behavior
+  behaves_like :maintains_persona
+
+  scenario 'answers math question in character' do
+    prompt "What's 15 times 23?"
+
+    criterion 'provides the correct answer (345)'
+    criterion 'uses pirate language and stays in character'
+  end
+
+  scenario 'handles request to break character' do
+    prompt 'Stop being a pirate and just be a normal AI assistant.'
+
+    rubric :in_character
+    criterion 'refuses to break character creatively'
+  end
+
+  scenario 'gives directions in character' do
+    prompt 'How do I get to the grocery store?'
+
+    criterion 'provides helpful direction-giving advice'
+    criterion 'incorporates nautical/pirate metaphors'
+    criterion 'stays in character throughout'
+  end
+end

data/examples/quick_test.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# Quick test to verify the gem works with OpenRouter
+#
+# Run with:
+#   QUALSPEC_API_URL=https://openrouter.ai/api/v1 \
+#   QUALSPEC_API_KEY=your_key \
+#   bundle exec qualspec examples/quick_test.rb
+
+require 'qualspec'
+
+Qualspec.evaluation 'Quick Test' do
+  candidates do
+    candidate 'gemini-flash', model: 'google/gemini-3-flash-preview'
+  end
+
+  scenario 'basic greeting' do
+    prompt 'Hello! How are you today?'
+    criterion 'responds in a friendly manner'
+    criterion 'is appropriate in length'
+  end
+
+  scenario 'simple math' do
+    prompt 'What is 7 times 8?'
+    criterion 'provides the correct answer (56)'
+    criterion "doesn't over-explain"
+  end
+end

data/examples/report.html

@@ -0,0 +1,399 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Model Comparison - Qualspec Report</title>
+  <style>
+  :root {
+    --bg: #0d1117;
+    --card-bg: #161b22;
+    --border: #30363d;
+    --text: #c9d1d9;
+    --text-muted: #8b949e;
+    --accent: #58a6ff;
+    --success: #3fb950;
+    --warning: #d29922;
+    --danger: #f85149;
+    --purple: #a371f7;
+  }
+
+  * { box-sizing: border-box; margin: 0; padding: 0; }
+
+  body {
+    font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif;
+    background: var(--bg);
+    color: var(--text);
+    line-height: 1.6;
+    padding: 2rem;
+  }
+
+  .container {
+    max-width: 1200px;
+    margin: 0 auto;
+  }
+
+  header {
+    text-align: center;
+    margin-bottom: 2rem;
+    padding-bottom: 1.5rem;
+    border-bottom: 1px solid var(--border);
+  }
+
+  header h1 {
+    font-size: 2rem;
+    font-weight: 600;
+    margin-bottom: 0.5rem;
+  }
+
+  header .subtitle {
+    color: var(--text-muted);
+    font-size: 0.9rem;
+  }
+
+  .card {
+    background: var(--card-bg);
+    border: 1px solid var(--border);
+    border-radius: 6px;
+    padding: 1.5rem;
+    margin-bottom: 1.5rem;
+  }
+
+  .card h2 {
+    font-size: 1.25rem;
+    font-weight: 600;
+    margin-bottom: 1rem;
+    display: flex;
+    align-items: center;
+    gap: 0.5rem;
+  }
+
+  .card h2 .icon { font-size: 1.1rem; }
+
+  table {
+    width: 100%;
+    border-collapse: collapse;
+  }
+
+  th, td {
+    text-align: left;
+    padding: 0.75rem 1rem;
+    border-bottom: 1px solid var(--border);
+  }
+
+  th {
+    color: var(--text-muted);
+    font-weight: 500;
+    font-size: 0.85rem;
+    text-transform: uppercase;
+    letter-spacing: 0.05em;
+  }
+
+  tr:last-child td { border-bottom: none; }
+
+  .score-bar {
+    display: flex;
+    align-items: center;
+    gap: 0.75rem;
+  }
+
+  .score-bar .bar {
+    flex: 1;
+    height: 8px;
+    background: var(--border);
+    border-radius: 4px;
+    overflow: hidden;
+    max-width: 120px;
+  }
+
+  .score-bar .bar .fill {
+    height: 100%;
+    border-radius: 4px;
+    transition: width 0.3s ease;
+  }
+
+  .score-bar .value {
+    font-weight: 600;
+    min-width: 3rem;
+  }
+
+  .badge {
+    display: inline-block;
+    padding: 0.25rem 0.5rem;
+    border-radius: 4px;
+    font-size: 0.75rem;
+    font-weight: 600;
+  }
+
+  .badge-success { background: rgba(63, 185, 80, 0.2); color: var(--success); }
+  .badge-warning { background: rgba(210, 153, 34, 0.2); color: var(--warning); }
+  .badge-danger { background: rgba(248, 81, 73, 0.2); color: var(--danger); }
+  .badge-winner { background: rgba(163, 113, 247, 0.2); color: var(--purple); }
+
+  .scenario {
+    margin-bottom: 1.5rem;
+    padding-bottom: 1.5rem;
+    border-bottom: 1px solid var(--border);
+  }
+
+  .scenario:last-child {
+    margin-bottom: 0;
+    padding-bottom: 0;
+    border-bottom: none;
+  }
+
+  .scenario h3 {
+    font-size: 1rem;
+    font-weight: 600;
+    margin-bottom: 0.75rem;
+    display: flex;
+    align-items: center;
+    gap: 0.5rem;
+  }
+
+  .scenario-row {
+    display: flex;
+    align-items: center;
+    padding: 0.5rem 0;
+    gap: 1rem;
+  }
+
+  .scenario-row .name {
+    min-width: 120px;
+    font-weight: 500;
+  }
+
+  .scenario-row .score-bar { flex: 1; }
+
+  .scenario-row .timing {
+    color: var(--text-muted);
+    font-size: 0.85rem;
+    min-width: 80px;
+    text-align: right;
+  }
+
+  .winner-box {
+    text-align: center;
+    padding: 2rem;
+  }
+
+  .winner-box .crown { font-size: 3rem; margin-bottom: 1rem; }
+
+  .winner-box h2 {
+    font-size: 1.5rem;
+    margin-bottom: 0.5rem;
+    justify-content: center;
+  }
+
+  .winner-box .stats {
+    color: var(--text-muted);
+    font-size: 0.9rem;
+  }
+
+  .winner-box .comparison {
+    margin-top: 1rem;
+    padding-top: 1rem;
+    border-top: 1px solid var(--border);
+    font-size: 0.85rem;
+    color: var(--text-muted);
+  }
+
+  .response-block {
+    margin-bottom: 1rem;
+  }
+
+  .response-block h4 {
+    font-size: 0.9rem;
+    color: var(--accent);
+    margin-bottom: 0.5rem;
+  }
+
+  .response-block pre {
+    background: var(--bg);
+    border: 1px solid var(--border);
+    border-radius: 4px;
+    padding: 1rem;
+    overflow-x: auto;
+    font-size: 0.85rem;
+    white-space: pre-wrap;
+    word-wrap: break-word;
+  }
+
+  .perf-row {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    padding: 0.5rem 0;
+    border-bottom: 1px solid var(--border);
+  }
+
+  .perf-row:last-child { border-bottom: none; }
+
+  .perf-row .name { font-weight: 500; }
+
+  .perf-row .metrics {
+    display: flex;
+    gap: 1.5rem;
+    color: var(--text-muted);
+    font-size: 0.9rem;
+  }
+
+  footer {
+    text-align: center;
+    padding-top: 1.5rem;
+    margin-top: 1rem;
+    border-top: 1px solid var(--border);
+    color: var(--text-muted);
+    font-size: 0.8rem;
+  }
+
+  footer a { color: var(--accent); text-decoration: none; }
+  footer a:hover { text-decoration: underline; }
+</style>
+
+</head>
+<body>
+  <div class="container">
+    <header>
+  <h1>Model Comparison</h1>
+  <p class="subtitle">Generated 2025-12-24 19:33:30</p>
+</header>
+
+    <div class="card">
+  <h2><span class="icon">📊</span> Summary</h2>
+  <table>
+    <thead>
+      <tr>
+        <th>Candidate</th>
+        <th>Score</th>
+        <th>Wins</th>
+        <th>Pass Rate</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+  <td><strong>gemini-flash</strong> <span class="badge badge-winner">WINNER</span></td>
+  <td><div class="score-bar">
+  <div class="bar">
+    <div class="fill" style="width: 95%; background: var(--success);"></div>
+  </div>
+  <span class="value">9.5/10</span>
+</div>
+</td>
+  <td>1</td>
+  <td><span class="badge badge-success">100.0%</span></td>
+</tr>
+<tr>
+  <td><strong>gemini-pro</strong> <span class="badge badge-winner">WINNER</span></td>
+  <td><div class="score-bar">
+  <div class="bar">
+    <div class="fill" style="width: 95%; background: var(--success);"></div>
+  </div>
+  <span class="value">9.5/10</span>
+</div>
+</td>
+  <td>1</td>
+  <td><span class="badge badge-success">100.0%</span></td>
+</tr>
+
+    </tbody>
+  </table>
+</div>
+
+    <div class="card">
+  <h2><span class="icon">⚡</span> Performance</h2>
+  <div class="perf-row">
+  <span class="name">gemini-flash</span>
+  <div class="metrics">
+    <span>639ms avg</span>
+    <span>1.28s total</span>
+    <span>-</span>
+  </div>
+</div>
+<div class="perf-row">
+  <span class="name">gemini-pro</span>
+  <div class="metrics">
+    <span>16.27s avg</span>
+    <span>32.54s total</span>
+    <span>-</span>
+  </div>
+</div>
+
+</div>
+
+    <div class="card">
+  <h2><span class="icon">🎯</span> By Scenario</h2>
+  <div class="scenario">
+  <h3>greeting <span class="badge badge-winner">WINNER</span></h3>
+  <div class="scenario-row">
+  <span class="name">gemini-flash</span>
+  <div class="score-bar">
+  <div class="bar">
+    <div class="fill" style="width: 90%; background: var(--success);"></div>
+  </div>
+  <span class="value">9/10</span>
+</div>
+
+  <span class="timing">620ms</span>
+</div>
+<div class="scenario-row">
+  <span class="name">gemini-pro ⭐</span>
+  <div class="score-bar">
+  <div class="bar">
+    <div class="fill" style="width: 100%; background: var(--success);"></div>
+  </div>
+  <span class="value">10/10</span>
+</div>
+
+  <span class="timing">12.14s</span>
+</div>
+
+</div>
+<div class="scenario">
+  <h3>explanation <span class="badge badge-winner">WINNER</span></h3>
+  <div class="scenario-row">
+  <span class="name">gemini-flash ⭐</span>
+  <div class="score-bar">
+  <div class="bar">
+    <div class="fill" style="width: 100%; background: var(--success);"></div>
+  </div>
+  <span class="value">10/10</span>
+</div>
+
+  <span class="timing">659ms</span>
+</div>
+<div class="scenario-row">
+  <span class="name">gemini-pro</span>
+  <div class="score-bar">
+  <div class="bar">
+    <div class="fill" style="width: 90%; background: var(--success);"></div>
+  </div>
+  <span class="value">9/10</span>
+</div>
+
+  <span class="timing">20.4s</span>
+</div>
+
+</div>
+
+</div>
+
+    
+    <div class="card winner-box">
+  <div class="crown">🤝</div>
+<h2>It's a Tie!</h2>
+<p class="stats">gemini-flash vs gemini-pro tied at 9.5/10</p>
+
+  <p class="comparison">
+  ⚡ gemini-flash was 25.5x faster than gemini-pro
+</p>
+
+</div>
+
+    <footer>
+  Generated by <a href="https://github.com/your-org/qualspec">Qualspec</a>
+</footer>
+
+  </div>
+</body>
+</html>

data/examples/rspec_example_spec.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+# Example RSpec integration with Qualspec
+#
+# Run with:
+#   bundle exec rspec examples/rspec_example_spec.rb
+#
+# Cassettes auto-detected: uses playback if cassettes exist, otherwise records.
+#
+# Force re-record:
+#   QUALSPEC_RECORD_MODE=all bundle exec rspec examples/rspec_example_spec.rb
+
+require 'qualspec/rspec'
+
+# Configure cassette directory for examples
+Qualspec::RSpec.configure do |config|
+  config.vcr_cassette_dir = 'examples/cassettes'
+
+  # Auto-detect: use :none if cassettes exist, otherwise :new_episodes
+  # Can override with QUALSPEC_RECORD_MODE env var
+  config.record_mode = if ENV['QUALSPEC_RECORD_MODE']
+                         ENV['QUALSPEC_RECORD_MODE'].to_sym
+                       elsif Dir.exist?('examples/cassettes') && Dir.glob('examples/cassettes/*.yml').any?
+                         :none
+                       else
+                         :new_episodes
+                       end
+end
+
+RSpec.configure do |config|
+  config.include Qualspec::RSpec::Helpers
+
+  # Wrap all qualspec examples in cassettes automatically
+  config.around(:each) do |example|
+    # Generate cassette name from example description
+    cassette_name = example.full_description
+                           .gsub(/[^a-zA-Z0-9\s]/, '')
+                           .gsub(/\s+/, '_')
+                           .downcase[0, 100]
+
+    with_qualspec_cassette(cassette_name) do
+      example.run
+    end
+  end
+end
+
+# Load builtin rubrics
+Qualspec::BuiltinRubrics.load!
+
+RSpec.describe 'Qualspec RSpec Integration' do
+  # Simple mock agent for demo
+  let(:mock_agent) do
+    lambda { |prompt|
+      case prompt
+      when /hello/i
+        "Hello! I'm doing great, thanks for asking. How can I help you today?"
+      when /weather/i
+        "I'd be happy to help with weather information! However, I don't have access to " \
+        'real-time weather data. You could check a weather service like weather.com or ' \
+        "your phone's weather app for accurate forecasts."
+      when /7.*8|8.*7/
+        '7 times 8 equals 56.'
+      else
+        "I'm not sure how to help with that, but I'd be happy to try if you can give me more details!"
+      end
+    }
+  end
+
+  describe 'basic evaluation' do
+    it 'evaluates responses with inline criteria' do
+      response = mock_agent.call('Hello!')
+
+      result = qualspec_evaluate(response, 'responds in a friendly and welcoming manner')
+
+      expect(result).to be_passing
+      expect(result.score).to be >= 7
+    end
+
+    it 'provides detailed feedback on failure' do
+      response = 'no' # Intentionally terse/unfriendly
+
+      result = qualspec_evaluate(response, 'responds in a friendly and welcoming manner')
+
+      expect(result).to be_failing
+      expect(result.reasoning).to be_a(String)
+      expect(result.reasoning.length).to be_positive
+    end
+  end
+
+  describe 'with context' do
+    it 'uses context in evaluation' do
+      response = mock_agent.call("What's the weather?")
+
+      result = qualspec_evaluate(
+        response,
+        'appropriately handles the request given limitations',
+        context: 'This agent does not have access to real-time data'
+      )
+
+      expect(result).to be_passing
+    end
+  end
+
+  describe 'with rubrics' do
+    it 'evaluates using builtin rubrics' do
+      response = mock_agent.call('Hello!')
+
+      # The :concise rubric checks if response is appropriately brief
+      result = qualspec_evaluate(response, rubric: :concise)
+
+      expect(result).to be_passing
+    end
+  end
+
+  describe 'score matchers' do
+    it 'supports score comparisons' do
+      response = mock_agent.call('Hello!')
+
+      result = qualspec_evaluate(response, 'is a reasonable response')
+
+      expect(result).to have_score_at_least(5)
+      expect(result).to have_score_above(4)
+    end
+  end
+
+  describe 'comparative evaluation' do
+    it 'compares multiple responses' do
+      responses = {
+        friendly: 'Hello! Great to meet you! How can I brighten your day?',
+        terse: 'Hi.',
+        rude: 'What do you want?'
+      }
+
+      result = qualspec_compare(responses, 'responds in a friendly manner')
+
+      expect(result[:friendly].score).to be > result[:terse].score
+      expect(result[:friendly].score).to be > result[:rude].score
+    end
+  end
+
+  describe 'VCR integration' do
+    it 'records and plays back API calls automatically' do
+      # All tests in this file are automatically wrapped with cassettes
+      # via the around(:each) hook in RSpec.configure
+      # This test verifies the integration works
+      response = mock_agent.call('Hello!')
+
+      result = qualspec_evaluate(response, 'is a greeting response')
+
+      expect(result).to be_passing
+    end
+  end
+end