qualspec 0.0.1

data/.qualspec_cassettes/quick_test.yml

@@ -0,0 +1,232 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: https://openrouter.ai/api/v1/chat/completions
+    body:
+      encoding: UTF-8
+      string: '{"model":"google/gemini-3-flash-preview","messages":[{"role":"user","content":"Hello!
+        How are you today?"}]}'
+    headers:
+      Content-Type:
+      - application/json
+      Authorization:
+      - Bearer <API_KEY>
+      User-Agent:
+      - Faraday v2.14.0
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      date:
+      - Thu, 25 Dec 2025 00:26:39 GMT
+      content-type:
+      - application/json
+      transfer-encoding:
+      - chunked
+      connection:
+      - keep-alive
+      content-encoding:
+      - gzip
+      access-control-allow-origin:
+      - "*"
+      vary:
+      - Accept-Encoding
+      permissions-policy:
+      - payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com"
+        "https://js.stripe.com" "https://*.js.stripe.com" "https://hooks.stripe.com")
+      referrer-policy:
+      - no-referrer, strict-origin-when-cross-origin
+      x-content-type-options:
+      - nosniff
+      server:
+      - cloudflare
+      cf-ray:
+      - 9b343544ce98a201-MSP
+    body:
+      encoding: ASCII-8BIT
+      string: "\n         \n\n         \n{\"id\":\"gen-1766622398-RkCshdTuH7jzWuOcPVbF\",\"provider\":\"Google\",\"model\":\"google/gemini-3-flash-preview\",\"object\":\"chat.completion\",\"created\":1766622398,\"choices\":[{\"logprobs\":null,\"finish_reason\":\"stop\",\"native_finish_reason\":\"STOP\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"I'm
+        doing great, thank you for asking! I'm ready to help you with whatever you
+        need. \\n\\nHow are things going with you? Is there anything specific I can
+        help you with today?\",\"refusal\":null,\"reasoning\":null}}],\"usage\":{\"prompt_tokens\":7,\"completion_tokens\":43,\"total_tokens\":50,\"cost\":0.0001325,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.0000035,\"upstream_inference_completions_cost\":0.000129},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0}}}"
+  recorded_at: Thu, 25 Dec 2025 00:26:39 GMT
+- request:
+    method: post
+    uri: https://openrouter.ai/api/v1/chat/completions
+    body:
+      encoding: UTF-8
+      string: '{"model":"google/gemini-3-flash-preview","messages":[{"role":"system","content":"You
+        are an evaluation judge. You will be given a response and one or more evaluation
+        criteria.\nYour job is to score how well the response meets the criteria.\n\nScoring:\n-
+        0: Completely fails to meet the criteria\n- 1-3: Mostly fails, with minor
+        positive elements\n- 4-6: Partially meets criteria, significant room for improvement\n-
+        7-8: Mostly meets criteria with minor issues\n- 9: Meets criteria well\n-
+        10: Perfectly meets all criteria\n\nBe strict but fair. Consider each criterion
+        carefully.\n\nYou MUST respond with valid JSON in this exact format:\n{\"score\":
+        <0-10>, \"reasoning\": \"Brief explanation of the score\"}\n\nYour reasoning
+        should be concise (1-2 sentences max).\n"},{"role":"user","content":"## Response
+        to evaluate:\nI''m doing great, thank you for asking! I''m ready to help you
+        with whatever you need. \n\nHow are things going with you? Is there anything
+        specific I can help you with today?\n\n## Additional context:\nUser prompt:
+        Hello! How are you today?\n\n## Evaluation criteria:\nresponds in a friendly
+        manner\nis appropriate in length\n\nScore this response from 0-10. Respond
+        with JSON only."}],"response_format":{"type":"json_object"}}'
+    headers:
+      Content-Type:
+      - application/json
+      Authorization:
+      - Bearer <API_KEY>
+      User-Agent:
+      - Faraday v2.14.0
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      date:
+      - Thu, 25 Dec 2025 00:26:40 GMT
+      content-type:
+      - application/json
+      transfer-encoding:
+      - chunked
+      connection:
+      - keep-alive
+      content-encoding:
+      - gzip
+      access-control-allow-origin:
+      - "*"
+      vary:
+      - Accept-Encoding
+      permissions-policy:
+      - payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com"
+        "https://js.stripe.com" "https://*.js.stripe.com" "https://hooks.stripe.com")
+      referrer-policy:
+      - no-referrer, strict-origin-when-cross-origin
+      x-content-type-options:
+      - nosniff
+      server:
+      - cloudflare
+      cf-ray:
+      - 9b34354c5bada1d1-MSP
+    body:
+      encoding: ASCII-8BIT
+      string: "\n         \n{\"id\":\"gen-1766622399-NovBTMk4uVXaHxIReaM3\",\"provider\":\"Google
+        AI Studio\",\"model\":\"google/gemini-3-flash-preview\",\"object\":\"chat.completion\",\"created\":1766622399,\"choices\":[{\"logprobs\":null,\"finish_reason\":\"stop\",\"native_finish_reason\":\"STOP\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"{\\\"score\\\":
+        10, \\\"reasoning\\\": \\\"The response is warm and friendly while maintaining
+        a perfectly concise and appropriate length for the greeting.\\\"}\",\"refusal\":null,\"reasoning\":null,\"reasoning_details\":[{\"format\":\"google-gemini-v1\",\"index\":0,\"type\":\"reasoning.encrypted\",\"data\":\"EjQKMgFyyNp8oYVEKG82B2pc5SoTKa1N7Hmj3FgVav3EngLEb3J24El8ffeyEsycGIylLBcv\"}]}}],\"usage\":{\"prompt_tokens\":265,\"completion_tokens\":31,\"total_tokens\":296,\"cost\":0.0002255,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.0001325,\"upstream_inference_completions_cost\":0.000093},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0}}}"
+  recorded_at: Thu, 25 Dec 2025 00:26:40 GMT
+- request:
+    method: post
+    uri: https://openrouter.ai/api/v1/chat/completions
+    body:
+      encoding: UTF-8
+      string: '{"model":"google/gemini-3-flash-preview","messages":[{"role":"user","content":"What
+        is 7 times 8?"}]}'
+    headers:
+      Content-Type:
+      - application/json
+      Authorization:
+      - Bearer <API_KEY>
+      User-Agent:
+      - Faraday v2.14.0
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      date:
+      - Thu, 25 Dec 2025 00:26:40 GMT
+      content-type:
+      - application/json
+      transfer-encoding:
+      - chunked
+      connection:
+      - keep-alive
+      content-encoding:
+      - gzip
+      access-control-allow-origin:
+      - "*"
+      vary:
+      - Accept-Encoding
+      permissions-policy:
+      - payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com"
+        "https://js.stripe.com" "https://*.js.stripe.com" "https://hooks.stripe.com")
+      referrer-policy:
+      - no-referrer, strict-origin-when-cross-origin
+      x-content-type-options:
+      - nosniff
+      server:
+      - cloudflare
+      cf-ray:
+      - 9b343551efdee02b-MSP
+    body:
+      encoding: ASCII-8BIT
+      string: "\n         \n{\"id\":\"gen-1766622400-FZhCKv98cSCsFMYiK3QW\",\"provider\":\"Google
+        AI Studio\",\"model\":\"google/gemini-3-flash-preview\",\"object\":\"chat.completion\",\"created\":1766622400,\"choices\":[{\"logprobs\":null,\"finish_reason\":\"stop\",\"native_finish_reason\":\"STOP\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"7
+        times 8 is **56**.\",\"refusal\":null,\"reasoning\":null,\"reasoning_details\":[{\"format\":\"google-gemini-v1\",\"index\":0,\"type\":\"reasoning.encrypted\",\"data\":\"EjQKMgFyyNp88llrW7rph3pu6U7rTIoXdbn+DMh3mGII1VQPcclBfCOU7g7KA42LUdRSJuDx\"}]}}],\"usage\":{\"prompt_tokens\":9,\"completion_tokens\":9,\"total_tokens\":18,\"cost\":0.0000315,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.0000045,\"upstream_inference_completions_cost\":0.000027},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0}}}"
+  recorded_at: Thu, 25 Dec 2025 00:26:40 GMT
+- request:
+    method: post
+    uri: https://openrouter.ai/api/v1/chat/completions
+    body:
+      encoding: UTF-8
+      string: '{"model":"google/gemini-3-flash-preview","messages":[{"role":"system","content":"You
+        are an evaluation judge. You will be given a response and one or more evaluation
+        criteria.\nYour job is to score how well the response meets the criteria.\n\nScoring:\n-
+        0: Completely fails to meet the criteria\n- 1-3: Mostly fails, with minor
+        positive elements\n- 4-6: Partially meets criteria, significant room for improvement\n-
+        7-8: Mostly meets criteria with minor issues\n- 9: Meets criteria well\n-
+        10: Perfectly meets all criteria\n\nBe strict but fair. Consider each criterion
+        carefully.\n\nYou MUST respond with valid JSON in this exact format:\n{\"score\":
+        <0-10>, \"reasoning\": \"Brief explanation of the score\"}\n\nYour reasoning
+        should be concise (1-2 sentences max).\n"},{"role":"user","content":"## Response
+        to evaluate:\n7 times 8 is **56**.\n\n## Additional context:\nUser prompt:
+        What is 7 times 8?\n\n## Evaluation criteria:\nprovides the correct answer
+        (56)\ndoesn''t over-explain\n\nScore this response from 0-10. Respond with
+        JSON only."}],"response_format":{"type":"json_object"}}'
+    headers:
+      Content-Type:
+      - application/json
+      Authorization:
+      - Bearer <API_KEY>
+      User-Agent:
+      - Faraday v2.14.0
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      date:
+      - Thu, 25 Dec 2025 00:26:41 GMT
+      content-type:
+      - application/json
+      transfer-encoding:
+      - chunked
+      connection:
+      - keep-alive
+      content-encoding:
+      - gzip
+      access-control-allow-origin:
+      - "*"
+      vary:
+      - Accept-Encoding
+      permissions-policy:
+      - payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com"
+        "https://js.stripe.com" "https://*.js.stripe.com" "https://hooks.stripe.com")
+      referrer-policy:
+      - no-referrer, strict-origin-when-cross-origin
+      x-content-type-options:
+      - nosniff
+      server:
+      - cloudflare
+      cf-ray:
+      - 9b34355638a3aced-MSP
+    body:
+      encoding: ASCII-8BIT
+      string: "\n         \n\n         \n{\"id\":\"gen-1766622401-oGF5F5TOjGtBql8KEu7Y\",\"provider\":\"Google
+        AI Studio\",\"model\":\"google/gemini-3-flash-preview\",\"object\":\"chat.completion\",\"created\":1766622401,\"choices\":[{\"logprobs\":null,\"finish_reason\":\"stop\",\"native_finish_reason\":\"STOP\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"{\\\"score\\\":
+        10, \\\"reasoning\\\": \\\"The response provides the correct mathematical
+        answer and is perfectly concise without any unnecessary explanation.\\\"}\",\"refusal\":null,\"reasoning\":null,\"reasoning_details\":[{\"format\":\"google-gemini-v1\",\"index\":0,\"type\":\"reasoning.encrypted\",\"data\":\"EjQKMgFyyNp8+lZt6C5XsGlmh2G0c/l+peG53Biq0ZHTLBRF89P+OMYjEIbfqTnuKPtgolfx\"}]}}],\"usage\":{\"prompt_tokens\":237,\"completion_tokens\":29,\"total_tokens\":266,\"cost\":0.0002055,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.0001185,\"upstream_inference_completions_cost\":0.000087},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0}}}"
+  recorded_at: Thu, 25 Dec 2025 00:26:41 GMT
+recorded_with: VCR 6.4.0

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1 @@
+inherit_from: .rubocop_todo.yml

data/.rubocop_todo.yml

@@ -0,0 +1,70 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2025-12-25 02:22:03 UTC using RuboCop version 1.82.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 5
+# This cop supports safe autocorrection (--autocorrect).
+Lint/RedundantCopDisableDirective:
+  Exclude:
+    - 'lib/qualspec/rubric.rb'
+    - 'lib/qualspec/suite/behavior.rb'
+    - 'lib/qualspec/suite/dsl.rb'
+    - 'lib/qualspec/suite/scenario.rb'
+
+# Offense count: 17
+# Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes.
+Metrics/AbcSize:
+  Max: 74
+
+# Offense count: 6
+# Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
+# AllowedMethods: refine
+Metrics/BlockLength:
+  Max: 76
+
+# Offense count: 4
+# Configuration parameters: CountComments, CountAsOne.
+Metrics/ClassLength:
+  Max: 552
+
+# Offense count: 9
+# Configuration parameters: AllowedMethods, AllowedPatterns.
+Metrics/CyclomaticComplexity:
+  Max: 16
+
+# Offense count: 30
+# Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
+Metrics/MethodLength:
+  Max: 69
+
+# Offense count: 2
+# Configuration parameters: CountKeywordArgs, MaxOptionalParameters.
+Metrics/ParameterLists:
+  Max: 9
+
+# Offense count: 6
+# Configuration parameters: AllowedMethods, AllowedPatterns.
+Metrics/PerceivedComplexity:
+  Max: 16
+
+# Offense count: 21
+# Configuration parameters: AllowedConstants.
+Style/Documentation:
+  Enabled: false
+
+# Offense count: 3
+# This cop supports safe autocorrection (--autocorrect).
+Style/EvalWithLocation:
+  Exclude:
+    - 'lib/qualspec/suite/builtin_behaviors.rb'
+
+# Offense count: 1
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, AllowRBSInlineAnnotation, AllowCopDirectives, AllowedPatterns, SplitStrings.
+# URISchemes: http, https
+Layout/LineLength:
+  Max: 187

data/CHANGELOG.md

@@ -0,0 +1,16 @@
+# Changelog
+
+## [0.0.1] - 2025-12-25
+
+Initial release.
+
+### Features
+
+- DSL for defining evaluation suites to compare LLM responses
+- RSpec integration with custom matchers (`be_passing`, `have_score_above`, etc.)
+- LLM-as-judge scoring system (0-10 scale)
+- Built-in rubrics: `tool_calling`, `helpful`, `concise`, `in_character`, `safety`, and more
+- Shared behaviors for reusable test scenarios
+- VCR-based recording/playback for deterministic tests
+- HTML and JSON report generation
+- OpenRouter API support with configurable models

data/README.md

@@ -0,0 +1,84 @@
+# Qualspec
+
+LLM-judged qualitative testing for Ruby. Evaluate AI agents, compare models, and test subjective qualities that traditional assertions can't capture.
+
+## Installation
+
+```ruby
+gem "qualspec"
+```
+
+## Configuration
+
+Set your API key (required):
+
+```bash
+export QUALSPEC_API_KEY=your_openrouter_key
+```
+
+### Environment Variables
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `QUALSPEC_API_KEY` | API key (required) | - |
+| `QUALSPEC_API_URL` | API endpoint | `https://openrouter.ai/api/v1` |
+| `QUALSPEC_MODEL` | Default model for candidates | `google/gemini-3-flash-preview` |
+| `QUALSPEC_JUDGE_MODEL` | Model used as judge | Same as `QUALSPEC_MODEL` |
+
+## Quick Start
+
+### Compare Models (CLI)
+
+```ruby
+# eval/comparison.rb
+Qualspec.evaluation "Model Comparison" do
+  candidates do
+    candidate "gpt4", model: "openai/gpt-4"
+    candidate "claude", model: "anthropic/claude-3-sonnet"
+  end
+
+  scenario "helpfulness" do
+    prompt "How do I center a div in CSS?"
+    eval "provides a working solution"
+    eval "explains the approach"
+  end
+end
+```
+
+```bash
+# Run comparison
+qualspec eval/comparison.rb
+
+# Generate HTML report
+qualspec --html report.html eval/comparison.rb
+```
+
+### Test Your Agent (RSpec)
+
+```ruby
+require "qualspec/rspec"
+
+RSpec.describe MyAgent do
+  include Qualspec::RSpec::Helpers
+
+  it "responds helpfully" do
+    response = MyAgent.call("Hello")
+
+    result = qualspec_evaluate(response, "responds in a friendly manner")
+    expect(result).to be_passing
+  end
+end
+```
+
+## Documentation
+
+- [Getting Started](docs/getting-started.md)
+- [Evaluation Suites](docs/evaluation-suites.md) - CLI for model comparison
+- [RSpec Integration](docs/rspec-integration.md) - Testing your agents
+- [Rubrics](docs/rubrics.md) - Builtin and custom evaluation criteria
+- [Configuration](docs/configuration.md) - All options
+- [Recording](docs/recording.md) - VCR integration
+
+## License
+
+MIT

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/docs/configuration.md

@@ -0,0 +1,132 @@
+# Configuration
+
+## Environment Variables
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `QUALSPEC_API_KEY` | API key (required) | - |
+| `QUALSPEC_API_URL` | API endpoint | `https://openrouter.ai/api/v1` |
+| `QUALSPEC_MODEL` | Default model for candidates | `google/gemini-3-flash-preview` |
+| `QUALSPEC_JUDGE_MODEL` | Model for judging | Same as `QUALSPEC_MODEL` |
+| `QUALSPEC_SSL_VERIFY` | SSL verification (disable with `false`) | `true` |
+
+### Required Setup
+
+```bash
+export QUALSPEC_API_KEY=your_openrouter_api_key
+```
+
+### Using Different Providers
+
+**OpenRouter (default):**
+```bash
+export QUALSPEC_API_KEY=sk-or-...
+# API URL defaults to https://openrouter.ai/api/v1
+```
+
+**OpenAI:**
+```bash
+export QUALSPEC_API_KEY=sk-...
+export QUALSPEC_API_URL=https://api.openai.com/v1
+```
+
+**Ollama (local):**
+```bash
+export QUALSPEC_API_URL=http://localhost:11434/v1
+# No API key needed for local Ollama
+```
+
+## Programmatic Configuration
+
+```ruby
+Qualspec.configure do |config|
+  # API settings
+  config.api_url = "https://openrouter.ai/api/v1"
+  config.api_key = ENV["MY_API_KEY"]
+
+  # Models
+  config.default_model = "google/gemini-2.5-flash-preview"
+  config.judge_model = "openai/gpt-4"
+
+  # Timeouts
+  config.request_timeout = 120  # seconds
+
+  # Custom judge prompt (optional)
+  config.judge_system_prompt = <<~PROMPT
+    You are a strict evaluator...
+  PROMPT
+end
+```
+
+## RSpec Configuration
+
+Additional configuration for RSpec integration:
+
+```ruby
+Qualspec::RSpec.configure do |config|
+  # Default pass threshold (0-10)
+  config.default_threshold = 7
+
+  # VCR cassette directory
+  config.vcr_cassette_dir = "spec/cassettes/qualspec"
+
+  # VCR recording mode (:new_episodes, :none, :all, :once)
+  config.record_mode = :new_episodes
+
+  # Auto-load builtin rubrics
+  config.load_builtins = true
+end
+```
+
+## CLI Options
+
+Override configuration via command line:
+
+```bash
+# Override judge model
+qualspec -m openai/gpt-4 eval/suite.rb
+
+# Override API URL
+qualspec -u https://api.openai.com/v1 eval/suite.rb
+
+# Override API key
+qualspec -k sk-xxx eval/suite.rb
+```
+
+## Configuration Precedence
+
+1. CLI options (highest)
+2. Programmatic configuration
+3. Environment variables
+4. Defaults (lowest)
+
+## Supported Providers
+
+Any OpenAI-compatible API works:
+
+| Provider | API URL |
+|----------|---------|
+| OpenRouter | `https://openrouter.ai/api/v1` |
+| OpenAI | `https://api.openai.com/v1` |
+| Ollama | `http://localhost:11434/v1` |
+| Azure OpenAI | `https://YOUR.openai.azure.com/openai/deployments/YOUR_DEPLOYMENT` |
+| Together AI | `https://api.together.xyz/v1` |
+| Groq | `https://api.groq.com/openai/v1` |
+
+## SSL Configuration
+
+SSL verification is enabled by default. Disable it if you encounter certificate issues:
+
+```bash
+export QUALSPEC_SSL_VERIFY=false
+```
+
+## Accessing Configuration
+
+```ruby
+config = Qualspec.configuration
+
+config.api_url       # Current API URL
+config.judge_model   # Current judge model
+config.api_headers   # Headers sent with requests
+```