qualspec 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '069d64a5f846962da842ffe5336739761862d0b1a3ef4cf522bfc83932af504e'
-  data.tar.gz: 693d58eb17d4e495cc0e7f1ff0ed57fc925d81ee6b8060aef41f350bf1fdb2c9
+  metadata.gz: 5fbaa225fce628f9becd02913d717112ec403d51e3d23a563c47ae285a7b0d4b
+  data.tar.gz: cfdb7943f93141c9377e809ebb384195ebcdf2ec3a9903c5a4b045b13d3eaef7
 SHA512:
-  metadata.gz: b46c32e6c43fb4f52234db0f4d6038191e85f8c05f99d8082c0024ef39fe0d7d1fa93a213b9163e8eef1c06a88add8a66f8906fa219ce8c471117a9812778766
-  data.tar.gz: de83bb65616448ccca96a405a021b19f80cbe9673df8cf65c63af24d2bdaa96efece1ea485b2786cfa7b3507f7f0ea301bd30d193c3f4218d18b67f3d2cfd695
+  metadata.gz: c76a9d16a428ff2ae61cb6b0274a1459e50c42aa9375ec966650f102030281e5c601654cb8323c71e845bfd86b0b51aa0a5d04c4487f32a0bf6c988bd9c73cd9
+  data.tar.gz: 6a21ec953e67ea7d7717d80de15c800fa0ac4eda3c2d24791543648312f71f436503f569296b138e743319258f41d6763160d10e50287befc9910e3fe10244d4

data/.rubocop.yml

@@ -1 +1,15 @@
 inherit_from: .rubocop_todo.yml
+
+inherit_mode:
+  merge:
+    - Exclude
+
+AllCops:
+  Exclude:
+    # Example scripts are illustrative docs, not library code.
+    - 'examples/**/*'
+
+# Spec files legitimately have long describe/context blocks.
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*'

data/.rubocop_todo.yml

@@ -34,7 +34,7 @@ Metrics/BlockLength:
 # Offense count: 5
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ClassLength:
-  Max: 552
+  Max: 560
 
 # Offense count: 12
 # Configuration parameters: AllowedMethods, AllowedPatterns.

data/CHANGELOG.md

@@ -1,5 +1,36 @@
 # Changelog
 
+## [0.2.0] - 2026-06-28
+
+### Added
+
+- **Named model registry** — curated models in `config/models.yml`, resolved via
+  `Qualspec.model(:name)` / `Qualspec.models`. Unknown/blank names fall back to
+  the default. Override the file with `QUALSPEC_MODELS_FILE`.
+- **Cost tracking (opt-in)** — `track_cost` in the suite DSL captures per-call
+  cost + tokens via OpenRouter usage accounting. `Results#value_ranking` and
+  `Results#cost_by_candidate` expose quality-per-dollar analysis (and raise a
+  helpful error if cost tracking wasn't enabled).
+- `Recorder.use_cassette` — replays an existing cassette (no API key needed),
+  records a fresh one when missing. Used by the new examples.
+- API key is now configurable via `Qualspec.configure { |c| c.api_key = ... }`,
+  falling back to `QUALSPEC_API_KEY` then `OPEN_ROUTER_API_KEY`.
+- New runnable examples: `customer_service_comparison`, `date_awareness_gate`,
+  `best_value`, `character_consistency` — each ships a committed VCR cassette.
+
+### Changed
+
+- Default model is now `openrouter/auto` (was `google/gemini-3-flash-preview`),
+  so qualspec works with nothing configured.
+- `candidate` no longer requires `model:` — it defaults to the configured model.
+- `Client#chat` requests usage accounting and reads cost from `usage.cost` when
+  `with_metadata: true` (previously cost was never captured).
+
+### Fixed
+
+- Cost data was never populated (metadata was never requested); cost reporting
+  now works when `track_cost` is enabled.
+
 ## [0.1.0] - 2025-12-26
 
 ### Added

data/README.md

@@ -10,7 +10,7 @@ gem "qualspec"
 
 ## Configuration
 
-Set your API key (required):
+Set your API key (`OPEN_ROUTER_API_KEY` also works as a fallback):
 
 ```bash
 export QUALSPEC_API_KEY=your_openrouter_key
@@ -20,10 +20,31 @@ export QUALSPEC_API_KEY=your_openrouter_key
 
 | Variable | Description | Default |
 |----------|-------------|---------|
-| `QUALSPEC_API_KEY` | API key (required) | - |
+| `QUALSPEC_API_KEY` | API key (required; falls back to `OPEN_ROUTER_API_KEY`) | - |
 | `QUALSPEC_API_URL` | API endpoint | `https://openrouter.ai/api/v1` |
-| `QUALSPEC_MODEL` | Default model for candidates | `google/gemini-3-flash-preview` |
+| `QUALSPEC_MODEL` | Default model for candidates | `openrouter/auto` |
 | `QUALSPEC_JUDGE_MODEL` | Model used as judge | Same as `QUALSPEC_MODEL` |
+| `QUALSPEC_MODELS_FILE` | Path to the named-models YAML | `config/models.yml` |
+
+### Models
+
+The default model everywhere is `openrouter/auto`, which routes to a sensible
+model for any request — so qualspec works even with nothing configured. A
+candidate with no `model:` uses this default too.
+
+Curated models live in `config/models.yml` and can be referenced by name:
+
+```ruby
+candidate :flash, model: Qualspec.model(:deepseek_flash)
+
+Qualspec.model(:glm)      # => "z-ai/glm-5.2"
+Qualspec.model(:unknown)  # => "openrouter/auto"  (falls back to default)
+Qualspec.model            # => "openrouter/auto"
+Qualspec.models.all       # => { "glm" => "z-ai/glm-5.2", ... }
+```
+
+Edit `config/models.yml` to add/rename models, or point `QUALSPEC_MODELS_FILE`
+at your own. Override the process-wide default with `QUALSPEC_MODEL`.
 
 ## Quick Start
 
@@ -73,11 +94,12 @@ end
 ## Documentation
 
 - [Getting Started](docs/getting-started.md)
-- [Evaluation Suites](docs/evaluation-suites.md) - CLI for model comparison
+- [Evaluation Suites](docs/evaluation-suites.md) - CLI for model comparison (incl. cost/value tracking)
 - [RSpec Integration](docs/rspec-integration.md) - Testing your agents
 - [Rubrics](docs/rubrics.md) - Builtin and custom evaluation criteria
-- [Configuration](docs/configuration.md) - All options
+- [Configuration](docs/configuration.md) - All options, models, cost tracking
 - [Recording](docs/recording.md) - VCR integration
+- [Examples](examples/EXAMPLES.md) - Runnable scripts (replay free from cassettes)
 
 ## License
 

data/config/models.yml

@@ -0,0 +1,23 @@
+# Curated OpenRouter models for qualspec.
+#
+# Reference these by name in suites/specs instead of hardcoding slugs:
+#
+#   candidate :flash, model: Qualspec.model(:deepseek_flash)
+#   Qualspec.model(:glm)        # => "z-ai/glm-5.2"
+#   Qualspec.model(:unknown)    # => falls back to `default` below
+#   Qualspec.model              # => the default
+#   Qualspec.models             # => the full name => slug hash
+#
+# `default` is the universal fallback. `openrouter/auto` routes to a sensible
+# model for any request, so things work even with nothing configured.
+# Override per-process with QUALSPEC_MODEL / QUALSPEC_JUDGE_MODEL, or point
+# QUALSPEC_MODELS_FILE at a different YAML file.
+default: openrouter/auto
+
+models:
+  owl_alpha:       openrouter/owl-alpha
+  deepseek_flash:  deepseek/deepseek-v4-flash
+  deepseek_pro:    deepseek/deepseek-v4-pro
+  minimax:         minimax/minimax-m3
+  glm:             z-ai/glm-5.2
+  gemini_flash:    google/gemini-3-flash-preview

data/docs/alpha_readiness.md

@@ -0,0 +1,94 @@
+# Qualspec — Alpha Readiness Findings & Plan
+
+_Assessment date: 2026-06-26. Status snapshot before first alpha use cases._
+
+> **Update 2026-06-28 (v0.2.0):** Several items below are now DONE:
+> - **Cost tracking fixed** (item 1) — now opt-in via `track_cost`; `Client` sends
+>   `usage: {include: true}` and reads `usage.cost`. Verified live + unit-tested.
+> - **API key configurable** — `Qualspec.configure { c.api_key = ... }` wins over
+>   `QUALSPEC_API_KEY` → `OPEN_ROUTER_API_KEY` (resolved lazily at read time).
+> - Added named model registry (`Qualspec.model`), `Recorder.use_cassette`, and a
+>   `Configuration` spec. Test count is now 74.
+> Still open: items 2–4 below (Client network-boundary spec, more pure units,
+> RSpec-integration spec) and the consistency pass for `api_url`/`default_model`.
+
+## TL;DR
+
+- **Tests: green** — `bundle exec rake spec` → 44 examples, 0 failures. RuboCop clean except 10 cosmetic offenses.
+- **No failing tests, no structural problems.** Architecture is clean (core / suite-DSL / rspec surfaces well separated).
+- **One real dead-code bug:** cost tracking never fires.
+- **~half of `lib/` has zero direct test coverage**, including the entire network boundary (`Client`) and the entire RSpec public surface.
+- **Open integration question:** we run a custom OpenRouter gem — `Client` currently talks to `chat/completions` via raw Faraday. Need to decide whether `Client` should delegate to our gem and confirm cost/header shapes line up.
+
+> Setup note: repo was locked to gems not installed for Ruby 3.4.9; ran `bundle install` to get green. Confirm alpha testers' Ruby matches `Gemfile.lock`.
+
+---
+
+## START HERE TOMORROW (ordered)
+
+### 0. Custom OpenRouter gem sync (do first — it changes item 1)
+The whole cost/metadata story depends on how our gem surfaces data, so resolve this before wiring cost.
+
+- `lib/qualspec/client.rb` is a self-contained Faraday client posting to `chat/completions`. Decide:
+  - **(a)** Keep `Client` as-is and just make sure our gem isn't double-wrapping requests, **or**
+  - **(b)** Have `Client` delegate to our OpenRouter gem.
+- Confirm the cost extraction in `Client#extract_cost` matches reality from our gem:
+  - currently reads header `x-openrouter-cost`, then falls back to body `usage.total_cost` / `cost` (`client.rb:110-117`).
+  - Verify our gem/endpoint actually returns one of these, and in what shape (header string vs. nested JSON).
+- Confirm token extraction keys (`prompt_tokens`/`completion_tokens`/`total_tokens`, `client.rb:119-128`) match.
+- Confirm auth headers (`config.api_headers`) and `response_format: { type: 'json_object' }` are honored by our endpoint (see item 4).
+
+### 1. Fix cost tracking (dead code) — depends on item 0
+**Bug:** `Candidate#generate_response` (`candidate.rb:22`) calls `Qualspec.client.chat(...)` **without** `with_metadata: true`, so `chat` always returns a `String`, never a `Client::Response`. Every `response.is_a?(Client::Response)` check in the runner (`runner.rb:72,82,83`) is permanently false. Result:
+- `cost:` always `nil` → `Results#costs` always empty
+- both reporters show cost as blank/`-` (`reporter.rb:147`, `html_reporter.rb:367`)
+- the `Response#duration_ms` path is dead (runner's own monotonic timer feeds timing, so timing itself is fine)
+
+**Fix:** thread `with_metadata: true` through `Candidate#generate_response` and have the runner consume the `Response` (cost + duration). Plumbing already exists in `Client#extract_cost`/`extract_tokens`. Alternative: remove the cost UI until wired. **Recommend wiring it.** (~30 min once item 0 is settled.)
+
+### 2. Add a `Client` spec (highest real-world risk) — ~1 hr
+The network boundary has **zero tests**. Use WebMock. Cover:
+- success path returns content
+- non-2xx → `Client::RequestError` (`handle_response`, `client.rb:85`)
+- `with_metadata: true` → builds `Response` with cost/tokens/duration
+- `extract_cost` header vs. body fallback
+- `validate_api_key!` raises when unset, skips during VCR playback
+- SSL toggle via `QUALSPEC_SSL_VERIFY=false`
+
+### 3. Pure-unit specs (high coverage-per-effort) — ~1 hr
+All currently untested, all pure/branchy:
+- `Candidate#normalize_temperature` — Anthropic clamps 0–1, others 0–2 (`candidate.rb:33`)
+- `Scenario#compose_prompt` / `compose_system_prompt` — priority merge `full_prompt > base_prompt > credential-prefix`; `variant > scenario > candidate` (`scenario.rb:60,84`)
+- `VariantsConfig#build_variants` / `#trait_matrix` — cartesian product, name dedup, **FactoryBot-absent fallback** (`dsl.rb:100,111`)
+- `PromptVariant` — `temperature=` range validation, `variant_key`, `customized?`, `to_h.compact` (`prompt_variant.rb:55,71`)
+
+### 4. One RSpec-integration spec — ~30 min
+Entire RSpec public surface (`helpers`, `matchers`, `evaluation_result`, `rspec/configuration`) is untested. Stub `Qualspec.judge`, exercise `qualspec_evaluate` + a couple matchers (`be_passing`, `have_score_above`). Also covers the symbol/string key fallback in `wrap_comparison_results` (`helpers.rb:143`).
+
+### 5. Small cleanups / docs
+- Drop duplicate `finish!` (called in `runner.rb:40` AND `qualspec.rb:81`).
+- Document in README: **scoring is comparative for 2+ candidates, absolute for 1** (`runner.rb:125`) — single-candidate and multi-candidate scores are NOT apples-to-apples.
+- `chmod +x` the two example files flagged by RuboCop.
+
+---
+
+## Full findings reference
+
+### Bugs / behavior gaps
+| # | Item | Location | Severity |
+|---|------|----------|----------|
+| 1 | Cost/metadata never captured (`with_metadata` never passed) | `candidate.rb:22` → `runner.rb:72,82,83` | **High** (dead feature) |
+| 2 | `finish!` called twice | `runner.rb:40`, `qualspec.rb:81` | Low |
+| 3 | Judge JSON-parse failure returns score 0 — indistinguishable from a genuinely bad response. Cross-provider endpoints that don't honor `json_object` will look like "model failed." | `judge.rb:162` | Medium (provider alpha) |
+| 4 | Comparative (2+) vs absolute (1) scoring — cross-run scores not comparable | `runner.rb:125` | Doc note |
+
+### Test coverage map
+**Has coverage:** `Judge` (happy + error), `Results`/`Runner` aggregation, `HtmlReporter`, top-level registry methods.
+
+**Zero coverage — ranked:**
+- **High:** `Client` (network boundary); entire RSpec integration (`helpers`/`matchers`/`evaluation_result`/`rspec/configuration`); `VariantsConfig` (headline feature); `Scenario#compose_prompt`.
+- **Medium:** `Candidate#normalize_temperature`; `PromptVariant` validation/keys; `Judge` untested branches (`winner: "tie"` `judge.rb:202`, `clamp(0,10)`, missing-candidate `judge.rb:190`).
+- **Low:** `Reporter` (stdout/json); CLI `exe/qualspec` smoke test; `Recorder`; builtin rubric/behavior content.
+
+### Effort estimate
+Items 0–5 ≈ **half a day to a day**. Items 1–4 take coverage from ~half to most of the meaningful surface and close the one real bug.

data/docs/configuration.md

@@ -4,16 +4,21 @@
 
 | Variable | Description | Default |
 |----------|-------------|---------|
-| `QUALSPEC_API_KEY` | API key (required) | - |
+| `QUALSPEC_API_KEY` | API key (falls back to `OPEN_ROUTER_API_KEY`) | - |
 | `QUALSPEC_API_URL` | API endpoint | `https://openrouter.ai/api/v1` |
-| `QUALSPEC_MODEL` | Default model for candidates | `google/gemini-3-flash-preview` |
+| `QUALSPEC_MODEL` | Default model for candidates | `openrouter/auto` |
 | `QUALSPEC_JUDGE_MODEL` | Model for judging | Same as `QUALSPEC_MODEL` |
+| `QUALSPEC_MODELS_FILE` | Path to the named-models YAML | `config/models.yml` |
 | `QUALSPEC_SSL_VERIFY` | SSL verification (disable with `false`) | `true` |
 
 ### Required Setup
 
+Provide an API key via env var (or set it programmatically — see below):
+
 ```bash
 export QUALSPEC_API_KEY=your_openrouter_api_key
+# or, equivalently for OpenRouter:
+export OPEN_ROUTER_API_KEY=sk-or-...
 ```
 
 ### Using Different Providers
@@ -42,10 +47,10 @@ export QUALSPEC_API_URL=http://localhost:11434/v1
 Qualspec.configure do |config|
   # API settings
   config.api_url = "https://openrouter.ai/api/v1"
-  config.api_key = ENV["MY_API_KEY"]
+  config.api_key = ENV["MY_API_KEY"]  # wins over QUALSPEC_API_KEY / OPEN_ROUTER_API_KEY
 
   # Models
-  config.default_model = "google/gemini-2.5-flash-preview"
+  config.default_model = "openrouter/auto"
   config.judge_model = "openai/gpt-4"
 
   # Timeouts
@@ -58,6 +63,50 @@ Qualspec.configure do |config|
 end
 ```
 
+## Models
+
+The default model everywhere is `openrouter/auto`, which routes to a sensible
+model for any request — so qualspec works even with nothing configured. A
+`candidate` with no `model:` uses this default.
+
+Curated models live in `config/models.yml` and are referenced by name:
+
+```yaml
+# config/models.yml
+default: openrouter/auto
+models:
+  glm:            z-ai/glm-5.2
+  deepseek_flash: deepseek/deepseek-v4-flash
+  deepseek_pro:   deepseek/deepseek-v4-pro
+```
+
+```ruby
+Qualspec.model(:glm)      # => "z-ai/glm-5.2"
+Qualspec.model(:unknown)  # => "openrouter/auto"  (falls back to default)
+Qualspec.model            # => "openrouter/auto"
+Qualspec.models.all       # => { "glm" => "z-ai/glm-5.2", ... }
+
+candidate :flash, model: Qualspec.model(:deepseek_flash)
+```
+
+Point `QUALSPEC_MODELS_FILE` at a different YAML to use your own list.
+
+## Cost Tracking
+
+Cost capture is **opt-in**. Enable `track_cost` in a suite to have qualspec
+request OpenRouter usage accounting and record per-call cost + tokens:
+
+```ruby
+Qualspec.evaluation "Best Value" do
+  track_cost
+  # ...
+end
+```
+
+Then `results.value_ranking` (quality per dollar) and `results.cost_by_candidate`
+become available. Calling them without `track_cost` raises a clear error. See
+[Evaluation Suites](evaluation-suites.md#cost-tracking) for details.
+
 ## RSpec Configuration
 
 Additional configuration for RSpec integration:

data/docs/evaluation-suites.md

@@ -29,8 +29,14 @@ candidates do
     system: "You are an extremely helpful assistant."
 
   # Multiple candidates
-  candidate "gemini", model: "google/gemini-2.5-flash-preview"
-  candidate "grok", model: "x-ai/grok-3-fast"
+  candidate "gemini", model: "google/gemini-3-flash-preview"
+  candidate "grok", model: "x-ai/grok-4.1-fast"
+
+  # Reference a curated model by name (see config/models.yml)
+  candidate "glm", model: Qualspec.model(:glm)
+
+  # Omit model: entirely to use the default (openrouter/auto)
+  candidate "auto"
 end
 ```
 
@@ -194,6 +200,43 @@ end
 
 This runs: **2 candidates × 4 variants × 3 temperatures × 1 scenario = 24 evaluations**
 
+## Cost Tracking
+
+Cost capture is opt-in. Add `track_cost` to a suite to record per-call cost and
+tokens (via OpenRouter usage accounting) so you can analyze the quality/cost
+trade-off:
+
+```ruby
+Qualspec.evaluation "Best Value" do
+  track_cost
+
+  candidates do
+    candidate :flash, model: Qualspec.model(:deepseek_flash)  # cheap
+    candidate :pro,   model: Qualspec.model(:deepseek_pro)    # pricey
+  end
+
+  scenario "reasoning task" do
+    prompt "A bat and ball cost $1.10. The bat is $1 more than the ball. How much is the ball?"
+    rubric :reasoning_quality
+  end
+end
+```
+
+After running, the `Results` object exposes cost analysis:
+
+```ruby
+results.costs_tracked?     # => true
+results.cost_by_candidate  # => { "flash" => 0.00014, "pro" => 0.00095 }
+
+# Rank by quality-per-dollar (avg score ÷ cost), best first:
+results.value_ranking
+# => { "flash" => { avg_score: 9.5, cost: 0.00014, score_per_dollar: 63982 }, ... }
+```
+
+Calling `value_ranking` or `cost_by_candidate` without `track_cost` raises a
+clear error telling you to enable it. See `examples/best_value.rb` for a full
+runnable example.
+
 ## Using Behaviors (Shared Scenarios)
 
 Define reusable scenario sets:

data/docs/getting-started.md

@@ -18,7 +18,7 @@ bundle install
 
 ## Configuration
 
-Set your API key:
+Set your API key (`OPEN_ROUTER_API_KEY` also works as a fallback):
 
 ```bash
 export QUALSPEC_API_KEY=your_openrouter_key
@@ -41,11 +41,14 @@ You can also configure programmatically:
 Qualspec.configure do |config|
   config.api_url = "https://openrouter.ai/api/v1"
   config.api_key = ENV["MY_API_KEY"]
-  config.judge_model = "google/gemini-2.5-flash-preview"
+  config.judge_model = "openrouter/auto"
   config.request_timeout = 120
 end
 ```
 
+Models can be referenced by name from `config/models.yml` via `Qualspec.model(:name)`,
+and the default everywhere is `openrouter/auto`. See [Configuration](configuration.md#models).
+
 ## Two Ways to Use Qualspec
 
 ### 1. Evaluation Suites (CLI)

data/docs/recording.md

@@ -29,6 +29,28 @@ qualspec --playback my_session eval/suite.rb
 
 Replays from the cassette with no network calls. Fails if a request isn't in the cassette.
 
+## Programmatic Recording
+
+For scripts (like the files in `examples/`), drive the recorder directly. Point
+it at a cassette directory, then use `use_cassette`, which **replays an existing
+cassette without needing an API key** and records a fresh one only when it's
+missing:
+
+```ruby
+Qualspec::Recorder.setup(cassette_dir: File.expand_path("cassettes", __dir__))
+
+Qualspec::Recorder.use_cassette("my_run") do
+  Qualspec.run("My Suite", output: :stdout)
+end
+```
+
+This is how the bundled examples ship committed cassettes that anyone can replay
+for free. To re-record, delete the cassette and run again with a key set.
+
+`Recorder.record(name)` (record new, replay existing) and
+`Recorder.playback(name)` (replay only, error on miss) are also available when
+you want to force a mode.
+
 ## RSpec Recording
 
 ### Per-Test Recording

data/examples/EXAMPLES.md

@@ -0,0 +1,73 @@
+# Qualspec Examples
+
+Runnable scripts that show different ways to use qualspec. Each one **records a
+VCR cassette** to `examples/cassettes/`, so they replay for free — no API key,
+no credits, no network:
+
+```bash
+bundle exec ruby examples/customer_service_comparison.rb
+```
+
+To run against the **live API** instead, delete the cassette and provide a key
+(the examples read `OPEN_ROUTER_API_KEY` / `QUALSPEC_API_KEY`):
+
+```bash
+rm examples/cassettes/customer_service_comparison.yml
+OPEN_ROUTER_API_KEY=sk-... bundle exec ruby examples/customer_service_comparison.rb
+```
+
+Cassettes are recorded with API keys filtered out (`<API_KEY>`). Models are
+referenced by name from `config/models.yml` via `Qualspec.model(:name)`.
+
+## The showcase
+
+| Example | Use case | What it demonstrates |
+|---------|----------|----------------------|
+| [`customer_service_comparison.rb`](customer_service_comparison.rb) | Pick the best model for a support agent | Suite DSL, multi-candidate **comparative judging**, per-scenario winners, custom rubric |
+| [`date_awareness_gate.rb`](date_awareness_gate.rb) | CI gate: fail if a model hallucinates | **Pass/fail thresholding** + non-zero exit code as a regression gate |
+| [`best_value.rb`](best_value.rb) | "Is the pricey model worth it?" | **Cost tracking** (`track_cost`) and `value_ranking` — quality per dollar |
+| [`character_consistency.rb`](character_consistency.rb) | Rank models on a multi-turn role-play | The **lower-level API** (`Qualspec.client` + `Qualspec.judge`) for multi-turn / agent evaluation |
+
+## What each one shows
+
+### `customer_service_comparison.rb` — compare models, find the best
+Three models answer the same support scenarios (an angry refund demand, an
+ambiguous discount question) under one system prompt. The judge compares them
+head-to-head and declares a per-scenario winner. This is the core
+"which model is best for this job?" workflow, driven entirely by the suite DSL
+and a custom `Qualspec.define_rubric`.
+
+### `date_awareness_gate.rb` — pass/fail gate for CI
+A single criterion: a model should *not* confidently assert today's date (it has
+no real-time clock). The script judges each model and **exits non-zero if any
+fails**, so you can drop it into CI as a guard when swapping models. In the
+recorded run, one model correctly declines while two confidently hallucinate a
+date — the gate fails (exit 1), as intended.
+
+### `best_value.rb` — best response per dollar
+Enables `track_cost`, which makes qualspec request OpenRouter usage accounting
+and capture real per-call cost. It then reports both the **quality winner** and
+the **value winner** via `results.value_ranking` (avg score ÷ cost). The
+recorded run shows a cheap model winning on value while the pro model costs
+several times more for a marginal quality gain.
+
+> Cost analysis requires `track_cost`. Calling `value_ranking` / `cost_by_candidate`
+> without it raises a clear error telling you to enable it.
+
+### `character_consistency.rb` — multi-turn ranking via the building blocks
+The suite DSL is single-prompt per scenario, so evaluating a *conversation* drops
+down to qualspec's primitives: `Qualspec.client.chat` with a running message
+history to role-play a game NPC over several turns, then
+`Qualspec.judge.evaluate_comparison` to rank the full transcripts on persona
+consistency. A good template for vetting an agent or a new model with a custom
+harness.
+
+## Other examples in this directory
+
+- `simple_variant_comparison.rb`, `variant_comparison.rb`,
+  `prompt_variants_factory.rb` — the **variant + temperature matrix** feature
+  (FactoryBot-backed prompt permutations).
+- `rspec_example_spec.rb` — using qualspec **inside RSpec** (`qualspec_evaluate`,
+  `qualspec_compare`, matchers, VCR).
+- `comparison.rb`, `model_comparison.rb`, `persona_test.rb`, `quick_test.rb` —
+  smaller/older one-off snippets.

data/examples/README.md

@@ -1,5 +1,10 @@
 # Qualspec Examples
 
+> **See [EXAMPLES.md](EXAMPLES.md) for the full showcase** — runnable scripts
+> for model comparison, pass/fail gates, cost/value analysis, and multi-turn
+> ranking, each replayable for free from a committed VCR cassette. The notes
+> below cover the variant-matrix examples specifically.
+
 ## Simple Variant Comparison
 
 Demonstrates multi-dimensional testing with prompt variants.

data/examples/best_value.rb

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Example: Best response-per-dollar (cost/value trade-off)
+# -------------------------------------------------------
+# Use case: "Is the expensive model actually worth it for this task?"
+#
+# Enables `track_cost`, which makes qualspec capture per-call cost from
+# OpenRouter's usage accounting. It then reports both the quality winner and the
+# value winner (avg score per dollar), so you can decide whether a pricier model
+# earns its keep.
+#
+# Run it (replays from cassette, no credits needed):
+#
+#     bundle exec ruby examples/best_value.rb
+
+require 'bundler/setup'
+require 'qualspec'
+
+Qualspec.define_rubric :reasoning_quality do
+  criterion 'Reaches the correct conclusion'
+  criterion 'Shows clear, valid step-by-step reasoning'
+  criterion 'Does not pad with irrelevant filler'
+end
+
+Qualspec.evaluation 'Best Value' do
+  track_cost # <-- capture per-call cost so value_ranking works
+
+  candidates do
+    candidate :flash, model: Qualspec.model(:deepseek_flash) # cheap
+    candidate :glm,   model: Qualspec.model(:glm)            # mid
+    candidate :pro,   model: Qualspec.model(:deepseek_pro)   # pricey
+  end
+
+  scenario 'Word problem' do
+    prompt 'A bat and a ball cost $1.10 total. The bat costs $1.00 more than the ball. ' \
+           'How much does the ball cost? Explain your reasoning.'
+    rubric :reasoning_quality
+  end
+
+  scenario 'Logic puzzle' do
+    prompt 'If all Bloops are Razzies and all Razzies are Lazzies, are all Bloops definitely Lazzies? ' \
+           'Explain.'
+    rubric :reasoning_quality
+  end
+end
+
+if __FILE__ == $PROGRAM_NAME
+  Qualspec::Recorder.setup(cassette_dir: File.expand_path('cassettes', __dir__))
+
+  results = Qualspec::Recorder.use_cassette('best_value') do
+    Qualspec.run('Best Value', progress: true, output: :stdout)
+  end
+
+  puts
+  puts '=== Value ranking (quality per dollar) ==='
+  printf("%-8s %8s %12s %14s\n", 'model', 'score', 'cost', 'score/$')
+  results.value_ranking.each do |candidate, v|
+    printf("%-8s %8.2f %12.6f %14s\n", candidate, v[:avg_score], v[:cost], v[:score_per_dollar] || 'n/a')
+  end
+
+  best_quality = results.scores_by_candidate.max_by { |_, s| s[:avg_score] }&.first
+  best_value   = results.value_ranking.keys.first
+  puts
+  puts "Highest quality: #{best_quality}"
+  puts "Best value:      #{best_value}"
+end