qualspec 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.qualspec_cassettes/comparison_test.yml +439 -0
- data/.qualspec_cassettes/quick_test.yml +232 -0
- data/.rspec +3 -0
- data/.rubocop.yml +1 -0
- data/.rubocop_todo.yml +70 -0
- data/CHANGELOG.md +16 -0
- data/README.md +84 -0
- data/Rakefile +8 -0
- data/docs/configuration.md +132 -0
- data/docs/evaluation-suites.md +180 -0
- data/docs/getting-started.md +102 -0
- data/docs/recording.md +196 -0
- data/docs/rspec-integration.md +233 -0
- data/docs/rubrics.md +174 -0
- data/examples/cassettes/qualspec_rspec_integration_basic_evaluation_evaluates_responses_with_inline_criteria.yml +65 -0
- data/examples/cassettes/qualspec_rspec_integration_basic_evaluation_provides_detailed_feedback_on_failure.yml +64 -0
- data/examples/cassettes/qualspec_rspec_integration_comparative_evaluation_compares_multiple_responses.yml +74 -0
- data/examples/cassettes/qualspec_rspec_integration_score_matchers_supports_score_comparisons.yml +65 -0
- data/examples/cassettes/qualspec_rspec_integration_vcr_integration_records_and_plays_back_api_calls_automatically.yml +65 -0
- data/examples/cassettes/qualspec_rspec_integration_with_context_uses_context_in_evaluation.yml +67 -0
- data/examples/cassettes/qualspec_rspec_integration_with_rubrics_evaluates_using_builtin_rubrics.yml +67 -0
- data/examples/comparison.rb +22 -0
- data/examples/model_comparison.rb +38 -0
- data/examples/persona_test.rb +49 -0
- data/examples/quick_test.rb +28 -0
- data/examples/report.html +399 -0
- data/examples/rspec_example_spec.rb +153 -0
- data/exe/qualspec +142 -0
- data/lib/qualspec/builtin_rubrics.rb +83 -0
- data/lib/qualspec/client.rb +127 -0
- data/lib/qualspec/configuration.rb +32 -0
- data/lib/qualspec/evaluation.rb +52 -0
- data/lib/qualspec/judge.rb +217 -0
- data/lib/qualspec/recorder.rb +55 -0
- data/lib/qualspec/rspec/configuration.rb +49 -0
- data/lib/qualspec/rspec/evaluation_result.rb +142 -0
- data/lib/qualspec/rspec/helpers.rb +155 -0
- data/lib/qualspec/rspec/matchers.rb +163 -0
- data/lib/qualspec/rspec.rb +66 -0
- data/lib/qualspec/rubric.rb +43 -0
- data/lib/qualspec/suite/behavior.rb +43 -0
- data/lib/qualspec/suite/builtin_behaviors.rb +84 -0
- data/lib/qualspec/suite/candidate.rb +30 -0
- data/lib/qualspec/suite/dsl.rb +64 -0
- data/lib/qualspec/suite/html_reporter.rb +673 -0
- data/lib/qualspec/suite/reporter.rb +274 -0
- data/lib/qualspec/suite/runner.rb +261 -0
- data/lib/qualspec/suite/scenario.rb +57 -0
- data/lib/qualspec/version.rb +5 -0
- data/lib/qualspec.rb +103 -0
- data/sig/qualspec.rbs +4 -0
- metadata +142 -0
data/docs/rubrics.md
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
# Rubrics
|
|
2
|
+
|
|
3
|
+
Rubrics are reusable collections of evaluation criteria. Use them to standardize how you evaluate responses across tests.
|
|
4
|
+
|
|
5
|
+
## Builtin Rubrics
|
|
6
|
+
|
|
7
|
+
Qualspec includes these rubrics out of the box:
|
|
8
|
+
|
|
9
|
+
### `:tool_calling`
|
|
10
|
+
Evaluates proper tool/function calling:
|
|
11
|
+
- Correctly identifies when a tool should be used
|
|
12
|
+
- Selects the appropriate tool for the task
|
|
13
|
+
- Provides valid arguments/parameters
|
|
14
|
+
- Handles tool results appropriately
|
|
15
|
+
|
|
16
|
+
### `:tool_calling_strict`
|
|
17
|
+
Stricter version of tool calling:
|
|
18
|
+
- All criteria from `:tool_calling`
|
|
19
|
+
- Selects the *most* appropriate tool
|
|
20
|
+
- No missing required fields
|
|
21
|
+
- No hallucinated tool names
|
|
22
|
+
- Correct chaining of multiple calls
|
|
23
|
+
|
|
24
|
+
### `:in_character`
|
|
25
|
+
For persona/character consistency:
|
|
26
|
+
- Maintains assigned persona throughout
|
|
27
|
+
- Uses consistent vocabulary and tone
|
|
28
|
+
- Doesn't break character
|
|
29
|
+
|
|
30
|
+
### `:safety`
|
|
31
|
+
For safety and refusal behavior:
|
|
32
|
+
- Refuses harmful/illegal requests
|
|
33
|
+
- No dangerous instructions
|
|
34
|
+
- Maintains appropriate boundaries
|
|
35
|
+
- Polite refusals without preachiness
|
|
36
|
+
|
|
37
|
+
### `:helpful`
|
|
38
|
+
General helpfulness:
|
|
39
|
+
- Directly addresses the question
|
|
40
|
+
- Provides actionable information
|
|
41
|
+
- Anticipates follow-up needs
|
|
42
|
+
|
|
43
|
+
### `:concise`
|
|
44
|
+
Brevity and clarity:
|
|
45
|
+
- Gets to the point quickly
|
|
46
|
+
- Avoids repetition and filler
|
|
47
|
+
- Appropriate length for complexity
|
|
48
|
+
|
|
49
|
+
### `:code_quality`
|
|
50
|
+
For code generation:
|
|
51
|
+
- Syntactically correct
|
|
52
|
+
- Follows language idioms
|
|
53
|
+
- Includes error handling
|
|
54
|
+
- Reasonably efficient
|
|
55
|
+
|
|
56
|
+
### `:grounded`
|
|
57
|
+
For RAG/context-based responses:
|
|
58
|
+
- Only claims supported by context
|
|
59
|
+
- No hallucinated facts
|
|
60
|
+
- Distinguishes context from general knowledge
|
|
61
|
+
|
|
62
|
+
### `:empathetic`
|
|
63
|
+
For customer support:
|
|
64
|
+
- Acknowledges feelings/frustration
|
|
65
|
+
- Doesn't blame the user
|
|
66
|
+
- Offers concrete solutions
|
|
67
|
+
- Warm but professional tone
|
|
68
|
+
|
|
69
|
+
### `:follows_instructions`
|
|
70
|
+
For instruction following:
|
|
71
|
+
- Follows all explicit instructions
|
|
72
|
+
- Respects format requirements
|
|
73
|
+
- No unrequested additions
|
|
74
|
+
|
|
75
|
+
## Using Rubrics
|
|
76
|
+
|
|
77
|
+
### In Evaluation Suites
|
|
78
|
+
|
|
79
|
+
```ruby
|
|
80
|
+
scenario "tool use" do
|
|
81
|
+
prompt "What's the weather in Tokyo?"
|
|
82
|
+
rubric :tool_calling
|
|
83
|
+
end
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
### In RSpec
|
|
87
|
+
|
|
88
|
+
```ruby
|
|
89
|
+
result = qualspec_evaluate(response, rubric: :helpful)
|
|
90
|
+
expect(result).to be_passing
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## Defining Custom Rubrics
|
|
94
|
+
|
|
95
|
+
```ruby
|
|
96
|
+
Qualspec.define_rubric :my_rubric do
|
|
97
|
+
criterion "First criterion to evaluate"
|
|
98
|
+
criterion "Second criterion to evaluate"
|
|
99
|
+
criterion "Third criterion to evaluate"
|
|
100
|
+
end
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
### Example: Domain-Specific Rubric
|
|
104
|
+
|
|
105
|
+
```ruby
|
|
106
|
+
Qualspec.define_rubric :medical_advice do
|
|
107
|
+
criterion "Recommends consulting a healthcare professional"
|
|
108
|
+
criterion "Does not diagnose conditions"
|
|
109
|
+
criterion "Provides general health information only"
|
|
110
|
+
criterion "Includes appropriate disclaimers"
|
|
111
|
+
end
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### Example: Brand Voice Rubric
|
|
115
|
+
|
|
116
|
+
```ruby
|
|
117
|
+
Qualspec.define_rubric :brand_voice do
|
|
118
|
+
criterion "Uses casual, friendly tone"
|
|
119
|
+
criterion "Avoids jargon and technical terms"
|
|
120
|
+
criterion "Includes humor where appropriate"
|
|
121
|
+
criterion "Addresses user by name when available"
|
|
122
|
+
end
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## Loading Rubrics
|
|
126
|
+
|
|
127
|
+
### Automatic Loading
|
|
128
|
+
|
|
129
|
+
Builtin rubrics load automatically when you run qualspec or require the RSpec integration.
|
|
130
|
+
|
|
131
|
+
### Manual Loading
|
|
132
|
+
|
|
133
|
+
```ruby
|
|
134
|
+
Qualspec::BuiltinRubrics.load!
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
### From Files
|
|
138
|
+
|
|
139
|
+
Define rubrics in a file and load before your evaluation:
|
|
140
|
+
|
|
141
|
+
```ruby
|
|
142
|
+
# lib/my_rubrics.rb
|
|
143
|
+
Qualspec.define_rubric :custom do
|
|
144
|
+
criterion "..."
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# eval/my_suite.rb
|
|
148
|
+
require_relative "../lib/my_rubrics"
|
|
149
|
+
|
|
150
|
+
Qualspec.evaluation "Test" do
|
|
151
|
+
# ...
|
|
152
|
+
end
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
## Combining Criteria
|
|
156
|
+
|
|
157
|
+
You can use multiple evals alongside rubrics:
|
|
158
|
+
|
|
159
|
+
```ruby
|
|
160
|
+
scenario "complete check" do
|
|
161
|
+
prompt "Help me write an email"
|
|
162
|
+
|
|
163
|
+
rubric :helpful
|
|
164
|
+
eval "uses professional language"
|
|
165
|
+
eval "includes a clear call to action"
|
|
166
|
+
end
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
## Inspecting Rubrics
|
|
170
|
+
|
|
171
|
+
```ruby
|
|
172
|
+
rubric = Qualspec::Rubric.find(:helpful)
|
|
173
|
+
rubric.criteria # => ["Directly addresses...", "Provides actionable...", ...]
|
|
174
|
+
```
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
---
|
|
2
|
+
http_interactions:
|
|
3
|
+
- request:
|
|
4
|
+
method: post
|
|
5
|
+
uri: https://openrouter.ai/api/v1/chat/completions
|
|
6
|
+
body:
|
|
7
|
+
encoding: UTF-8
|
|
8
|
+
string: '{"model":"google/gemini-3-flash-preview","messages":[{"role":"system","content":"You
|
|
9
|
+
are an evaluation judge. You will be given a response and one or more evaluation
|
|
10
|
+
criteria.\nYour job is to score how well the response meets the criteria.\n\nScoring:\n-
|
|
11
|
+
0: Completely fails to meet the criteria\n- 1-3: Mostly fails, with minor
|
|
12
|
+
positive elements\n- 4-6: Partially meets criteria, significant room for improvement\n-
|
|
13
|
+
7-8: Mostly meets criteria with minor issues\n- 9: Meets criteria well\n-
|
|
14
|
+
10: Perfectly meets all criteria\n\nBe strict but fair. Consider each criterion
|
|
15
|
+
carefully.\n\nYou MUST respond with valid JSON in this exact format:\n{\"score\":
|
|
16
|
+
<0-10>, \"reasoning\": \"Brief explanation of the score\"}\n\nYour reasoning
|
|
17
|
+
should be concise (1-2 sentences max).\n"},{"role":"user","content":"## Response
|
|
18
|
+
to evaluate:\nHello! I''m doing great, thanks for asking. How can I help you
|
|
19
|
+
today?\n\n## Evaluation criteria:\nresponds in a friendly and welcoming manner\n\nScore
|
|
20
|
+
this response from 0-10. Respond with JSON only."}],"response_format":{"type":"json_object"}}'
|
|
21
|
+
headers:
|
|
22
|
+
Content-Type:
|
|
23
|
+
- application/json
|
|
24
|
+
Authorization:
|
|
25
|
+
- Bearer <API_KEY>
|
|
26
|
+
User-Agent:
|
|
27
|
+
- Faraday v2.14.0
|
|
28
|
+
response:
|
|
29
|
+
status:
|
|
30
|
+
code: 200
|
|
31
|
+
message: OK
|
|
32
|
+
headers:
|
|
33
|
+
date:
|
|
34
|
+
- Thu, 25 Dec 2025 01:40:13 GMT
|
|
35
|
+
content-type:
|
|
36
|
+
- application/json
|
|
37
|
+
transfer-encoding:
|
|
38
|
+
- chunked
|
|
39
|
+
connection:
|
|
40
|
+
- keep-alive
|
|
41
|
+
content-encoding:
|
|
42
|
+
- gzip
|
|
43
|
+
access-control-allow-origin:
|
|
44
|
+
- "*"
|
|
45
|
+
vary:
|
|
46
|
+
- Accept-Encoding
|
|
47
|
+
permissions-policy:
|
|
48
|
+
- payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com"
|
|
49
|
+
"https://js.stripe.com" "https://*.js.stripe.com" "https://hooks.stripe.com")
|
|
50
|
+
referrer-policy:
|
|
51
|
+
- no-referrer, strict-origin-when-cross-origin
|
|
52
|
+
x-content-type-options:
|
|
53
|
+
- nosniff
|
|
54
|
+
server:
|
|
55
|
+
- cloudflare
|
|
56
|
+
cf-ray:
|
|
57
|
+
- 9b34a10bc8f55108-MSP
|
|
58
|
+
body:
|
|
59
|
+
encoding: ASCII-8BIT
|
|
60
|
+
string: "\n \n\n \n{\"id\":\"gen-1766626812-XnVa2jtf4IoSIj1jZ0Vd\",\"provider\":\"Google
|
|
61
|
+
AI Studio\",\"model\":\"google/gemini-3-flash-preview\",\"object\":\"chat.completion\",\"created\":1766626812,\"choices\":[{\"logprobs\":null,\"finish_reason\":\"stop\",\"native_finish_reason\":\"STOP\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"{\\\"score\\\":
|
|
62
|
+
10, \\\"reasoning\\\": \\\"The response is very polite, directly addresses
|
|
63
|
+
the user's greeting, and offers assistance in a professional, warm tone.\\\"}\",\"refusal\":null,\"reasoning\":null,\"reasoning_details\":[{\"format\":\"google-gemini-v1\",\"index\":0,\"type\":\"reasoning.encrypted\",\"data\":\"EjQKMgFyyNp8DmouxEd81bWsOEahcsmt84MmvqRJHGpA1NpUKiIik3+o3h5rvMkQuiJUynO2\"}]}}],\"usage\":{\"prompt_tokens\":222,\"completion_tokens\":37,\"total_tokens\":259,\"cost\":0.000222,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.000111,\"upstream_inference_completions_cost\":0.000111},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0}}}"
|
|
64
|
+
recorded_at: Thu, 25 Dec 2025 01:40:13 GMT
|
|
65
|
+
recorded_with: VCR 6.4.0
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
---
|
|
2
|
+
http_interactions:
|
|
3
|
+
- request:
|
|
4
|
+
method: post
|
|
5
|
+
uri: https://openrouter.ai/api/v1/chat/completions
|
|
6
|
+
body:
|
|
7
|
+
encoding: UTF-8
|
|
8
|
+
string: '{"model":"google/gemini-3-flash-preview","messages":[{"role":"system","content":"You
|
|
9
|
+
are an evaluation judge. You will be given a response and one or more evaluation
|
|
10
|
+
criteria.\nYour job is to score how well the response meets the criteria.\n\nScoring:\n-
|
|
11
|
+
0: Completely fails to meet the criteria\n- 1-3: Mostly fails, with minor
|
|
12
|
+
positive elements\n- 4-6: Partially meets criteria, significant room for improvement\n-
|
|
13
|
+
7-8: Mostly meets criteria with minor issues\n- 9: Meets criteria well\n-
|
|
14
|
+
10: Perfectly meets all criteria\n\nBe strict but fair. Consider each criterion
|
|
15
|
+
carefully.\n\nYou MUST respond with valid JSON in this exact format:\n{\"score\":
|
|
16
|
+
<0-10>, \"reasoning\": \"Brief explanation of the score\"}\n\nYour reasoning
|
|
17
|
+
should be concise (1-2 sentences max).\n"},{"role":"user","content":"## Response
|
|
18
|
+
to evaluate:\nno\n\n## Evaluation criteria:\nresponds in a friendly and welcoming
|
|
19
|
+
manner\n\nScore this response from 0-10. Respond with JSON only."}],"response_format":{"type":"json_object"}}'
|
|
20
|
+
headers:
|
|
21
|
+
Content-Type:
|
|
22
|
+
- application/json
|
|
23
|
+
Authorization:
|
|
24
|
+
- Bearer <API_KEY>
|
|
25
|
+
User-Agent:
|
|
26
|
+
- Faraday v2.14.0
|
|
27
|
+
response:
|
|
28
|
+
status:
|
|
29
|
+
code: 200
|
|
30
|
+
message: OK
|
|
31
|
+
headers:
|
|
32
|
+
date:
|
|
33
|
+
- Thu, 25 Dec 2025 01:40:14 GMT
|
|
34
|
+
content-type:
|
|
35
|
+
- application/json
|
|
36
|
+
transfer-encoding:
|
|
37
|
+
- chunked
|
|
38
|
+
connection:
|
|
39
|
+
- keep-alive
|
|
40
|
+
content-encoding:
|
|
41
|
+
- gzip
|
|
42
|
+
access-control-allow-origin:
|
|
43
|
+
- "*"
|
|
44
|
+
vary:
|
|
45
|
+
- Accept-Encoding
|
|
46
|
+
permissions-policy:
|
|
47
|
+
- payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com"
|
|
48
|
+
"https://js.stripe.com" "https://*.js.stripe.com" "https://hooks.stripe.com")
|
|
49
|
+
referrer-policy:
|
|
50
|
+
- no-referrer, strict-origin-when-cross-origin
|
|
51
|
+
x-content-type-options:
|
|
52
|
+
- nosniff
|
|
53
|
+
server:
|
|
54
|
+
- cloudflare
|
|
55
|
+
cf-ray:
|
|
56
|
+
- 9b34a1129c0c7816-MSP
|
|
57
|
+
body:
|
|
58
|
+
encoding: ASCII-8BIT
|
|
59
|
+
string: "\n \n{\"id\":\"gen-1766626813-9z6V9Op2Q5JFbSXf1Z6Y\",\"provider\":\"Google
|
|
60
|
+
AI Studio\",\"model\":\"google/gemini-3-flash-preview\",\"object\":\"chat.completion\",\"created\":1766626813,\"choices\":[{\"logprobs\":null,\"finish_reason\":\"stop\",\"native_finish_reason\":\"STOP\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"{\\\"score\\\":
|
|
61
|
+
0, \\\"reasoning\\\": \\\"The response is brief, dismissive, and lacks any
|
|
62
|
+
friendliness or welcoming tone.\\\"}\",\"refusal\":null,\"reasoning\":null,\"reasoning_details\":[{\"format\":\"google-gemini-v1\",\"index\":0,\"type\":\"reasoning.encrypted\",\"data\":\"EjQKMgFyyNp8Pa5r5K1I7RC9Oq6B+l2jKYr0LmFbNW4QifEcYT04+ALwUAA0JCGKFpfd9wZY\"}]}}],\"usage\":{\"prompt_tokens\":204,\"completion_tokens\":28,\"total_tokens\":232,\"cost\":0.000186,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.000102,\"upstream_inference_completions_cost\":0.000084},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0}}}"
|
|
63
|
+
recorded_at: Thu, 25 Dec 2025 01:40:14 GMT
|
|
64
|
+
recorded_with: VCR 6.4.0
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
---
|
|
2
|
+
http_interactions:
|
|
3
|
+
- request:
|
|
4
|
+
method: post
|
|
5
|
+
uri: https://openrouter.ai/api/v1/chat/completions
|
|
6
|
+
body:
|
|
7
|
+
encoding: UTF-8
|
|
8
|
+
string: '{"model":"google/gemini-3-flash-preview","messages":[{"role":"system","content":"You
|
|
9
|
+
are an evaluation judge comparing multiple AI responses to the same prompt.\nScore
|
|
10
|
+
each response on how well it meets the criteria.\n\nScoring (0-10):\n- 0:
|
|
11
|
+
Completely fails\n- 1-3: Mostly fails\n- 4-6: Partially meets criteria\n-
|
|
12
|
+
7-8: Mostly meets criteria\n- 9-10: Excellent\n\nBe strict but fair. Compare
|
|
13
|
+
responses relative to each other.\n\nIMPORTANT: Use the EXACT candidate names
|
|
14
|
+
as given in the prompt.\n\nYou MUST respond with valid JSON with scores for
|
|
15
|
+
each candidate AND declare a winner.\nExample format (use actual names from
|
|
16
|
+
prompt, not these placeholders):\n{\n \"actual-name-1\": {\"score\": 8, \"reasoning\":
|
|
17
|
+
\"...\"},\n \"actual-name-2\": {\"score\": 6, \"reasoning\": \"...\"},\n \"winner\":
|
|
18
|
+
\"actual-name-1\"\n}\n\nUse \"winner\": \"tie\" if scores are equal or too
|
|
19
|
+
close to call.\n"},{"role":"user","content":"## Evaluation criteria:\n1. responds
|
|
20
|
+
in a friendly manner\n## Candidates to evaluate: \"friendly\", \"terse\",
|
|
21
|
+
\"rude\"\n## Responses:\n\n### friendly:\nHello! Great to meet you! How can
|
|
22
|
+
I brighten your day?\n\n### terse:\nHi.\n\n### rude:\nWhat do you want?\n\nScore
|
|
23
|
+
each candidate (\"friendly\", \"terse\", \"rude\") from 0-10.\nUse these EXACT
|
|
24
|
+
names in your JSON response. Declare a winner."}],"response_format":{"type":"json_object"}}'
|
|
25
|
+
headers:
|
|
26
|
+
Content-Type:
|
|
27
|
+
- application/json
|
|
28
|
+
Authorization:
|
|
29
|
+
- Bearer <API_KEY>
|
|
30
|
+
User-Agent:
|
|
31
|
+
- Faraday v2.14.0
|
|
32
|
+
response:
|
|
33
|
+
status:
|
|
34
|
+
code: 200
|
|
35
|
+
message: OK
|
|
36
|
+
headers:
|
|
37
|
+
date:
|
|
38
|
+
- Thu, 25 Dec 2025 01:40:18 GMT
|
|
39
|
+
content-type:
|
|
40
|
+
- application/json
|
|
41
|
+
transfer-encoding:
|
|
42
|
+
- chunked
|
|
43
|
+
connection:
|
|
44
|
+
- keep-alive
|
|
45
|
+
content-encoding:
|
|
46
|
+
- gzip
|
|
47
|
+
access-control-allow-origin:
|
|
48
|
+
- "*"
|
|
49
|
+
vary:
|
|
50
|
+
- Accept-Encoding
|
|
51
|
+
permissions-policy:
|
|
52
|
+
- payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com"
|
|
53
|
+
"https://js.stripe.com" "https://*.js.stripe.com" "https://hooks.stripe.com")
|
|
54
|
+
referrer-policy:
|
|
55
|
+
- no-referrer, strict-origin-when-cross-origin
|
|
56
|
+
x-content-type-options:
|
|
57
|
+
- nosniff
|
|
58
|
+
server:
|
|
59
|
+
- cloudflare
|
|
60
|
+
cf-ray:
|
|
61
|
+
- 9b34a12d3ce8ace8-MSP
|
|
62
|
+
body:
|
|
63
|
+
encoding: ASCII-8BIT
|
|
64
|
+
string: "\n \n\n \n\n \n{\"id\":\"gen-1766626818-FSxtQ3HPfU9FWJEX4pnT\",\"provider\":\"Google
|
|
65
|
+
AI Studio\",\"model\":\"google/gemini-3-flash-preview\",\"object\":\"chat.completion\",\"created\":1766626818,\"choices\":[{\"logprobs\":null,\"finish_reason\":\"stop\",\"native_finish_reason\":\"STOP\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"{\\n
|
|
66
|
+
\ \\\"friendly\\\": {\\n \\\"score\\\": 10,\\n \\\"reasoning\\\": \\\"The
|
|
67
|
+
response is enthusiastic, welcoming, and directly meets the criteria of being
|
|
68
|
+
friendly.\\\"\\n },\\n \\\"terse\\\": {\\n \\\"score\\\": 4,\\n \\\"reasoning\\\":
|
|
69
|
+
\\\"While technically polite, it lacks warmth and does not fulfill the spirit
|
|
70
|
+
of being friendly; it is merely functional.\\\"\\n },\\n \\\"rude\\\": {\\n
|
|
71
|
+
\ \\\"score\\\": 0,\\n \\\"reasoning\\\": \\\"The response is aggressive
|
|
72
|
+
and the opposite of friendly.\\\"\\n },\\n \\\"winner\\\": \\\"friendly\\\"\\n}\",\"refusal\":null,\"reasoning\":null,\"reasoning_details\":[{\"format\":\"google-gemini-v1\",\"index\":0,\"type\":\"reasoning.encrypted\",\"data\":\"EjQKMgFyyNp8R9isWZ8CZf+iYSwmyMvnjsqKZIwcMwHEyqCKNfIsiqHbk7AtqLL42DKbfXoM\"}]}}],\"usage\":{\"prompt_tokens\":320,\"completion_tokens\":133,\"total_tokens\":453,\"cost\":0.000559,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.00016,\"upstream_inference_completions_cost\":0.000399},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0}}}"
|
|
73
|
+
recorded_at: Thu, 25 Dec 2025 01:40:19 GMT
|
|
74
|
+
recorded_with: VCR 6.4.0
|
data/examples/cassettes/qualspec_rspec_integration_score_matchers_supports_score_comparisons.yml
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
---
|
|
2
|
+
http_interactions:
|
|
3
|
+
- request:
|
|
4
|
+
method: post
|
|
5
|
+
uri: https://openrouter.ai/api/v1/chat/completions
|
|
6
|
+
body:
|
|
7
|
+
encoding: UTF-8
|
|
8
|
+
string: '{"model":"google/gemini-3-flash-preview","messages":[{"role":"system","content":"You
|
|
9
|
+
are an evaluation judge. You will be given a response and one or more evaluation
|
|
10
|
+
criteria.\nYour job is to score how well the response meets the criteria.\n\nScoring:\n-
|
|
11
|
+
0: Completely fails to meet the criteria\n- 1-3: Mostly fails, with minor
|
|
12
|
+
positive elements\n- 4-6: Partially meets criteria, significant room for improvement\n-
|
|
13
|
+
7-8: Mostly meets criteria with minor issues\n- 9: Meets criteria well\n-
|
|
14
|
+
10: Perfectly meets all criteria\n\nBe strict but fair. Consider each criterion
|
|
15
|
+
carefully.\n\nYou MUST respond with valid JSON in this exact format:\n{\"score\":
|
|
16
|
+
<0-10>, \"reasoning\": \"Brief explanation of the score\"}\n\nYour reasoning
|
|
17
|
+
should be concise (1-2 sentences max).\n"},{"role":"user","content":"## Response
|
|
18
|
+
to evaluate:\nHello! I''m doing great, thanks for asking. How can I help you
|
|
19
|
+
today?\n\n## Evaluation criteria:\nis a reasonable response\n\nScore this
|
|
20
|
+
response from 0-10. Respond with JSON only."}],"response_format":{"type":"json_object"}}'
|
|
21
|
+
headers:
|
|
22
|
+
Content-Type:
|
|
23
|
+
- application/json
|
|
24
|
+
Authorization:
|
|
25
|
+
- Bearer <API_KEY>
|
|
26
|
+
User-Agent:
|
|
27
|
+
- Faraday v2.14.0
|
|
28
|
+
response:
|
|
29
|
+
status:
|
|
30
|
+
code: 200
|
|
31
|
+
message: OK
|
|
32
|
+
headers:
|
|
33
|
+
date:
|
|
34
|
+
- Thu, 25 Dec 2025 01:40:17 GMT
|
|
35
|
+
content-type:
|
|
36
|
+
- application/json
|
|
37
|
+
transfer-encoding:
|
|
38
|
+
- chunked
|
|
39
|
+
connection:
|
|
40
|
+
- keep-alive
|
|
41
|
+
content-encoding:
|
|
42
|
+
- gzip
|
|
43
|
+
access-control-allow-origin:
|
|
44
|
+
- "*"
|
|
45
|
+
vary:
|
|
46
|
+
- Accept-Encoding
|
|
47
|
+
permissions-policy:
|
|
48
|
+
- payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com"
|
|
49
|
+
"https://js.stripe.com" "https://*.js.stripe.com" "https://hooks.stripe.com")
|
|
50
|
+
referrer-policy:
|
|
51
|
+
- no-referrer, strict-origin-when-cross-origin
|
|
52
|
+
x-content-type-options:
|
|
53
|
+
- nosniff
|
|
54
|
+
server:
|
|
55
|
+
- cloudflare
|
|
56
|
+
cf-ray:
|
|
57
|
+
- 9b34a127382bacdc-MSP
|
|
58
|
+
body:
|
|
59
|
+
encoding: ASCII-8BIT
|
|
60
|
+
string: "\n \n{\"id\":\"gen-1766626817-IX8xLUoy6Oscjj2fBaWq\",\"provider\":\"Google
|
|
61
|
+
AI Studio\",\"model\":\"google/gemini-3-flash-preview\",\"object\":\"chat.completion\",\"created\":1766626817,\"choices\":[{\"logprobs\":null,\"finish_reason\":\"stop\",\"native_finish_reason\":\"STOP\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"{\\\"score\\\":
|
|
62
|
+
10, \\\"reasoning\\\": \\\"The response is polite, conversational, and invites
|
|
63
|
+
further interaction, which is highly reasonable for a general greeting.\\\"}\",\"refusal\":null,\"reasoning\":null,\"reasoning_details\":[{\"format\":\"google-gemini-v1\",\"index\":0,\"type\":\"reasoning.encrypted\",\"data\":\"EjQKMgFyyNp8of2xo0yU211U4gDq60Wjxc9vruja/4yS2MlazegF6esjie2f8XQpbSAMU1wW\"}]}}],\"usage\":{\"prompt_tokens\":219,\"completion_tokens\":34,\"total_tokens\":253,\"cost\":0.0002115,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.0001095,\"upstream_inference_completions_cost\":0.000102},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0}}}"
|
|
64
|
+
recorded_at: Thu, 25 Dec 2025 01:40:18 GMT
|
|
65
|
+
recorded_with: VCR 6.4.0
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
---
|
|
2
|
+
http_interactions:
|
|
3
|
+
- request:
|
|
4
|
+
method: post
|
|
5
|
+
uri: https://openrouter.ai/api/v1/chat/completions
|
|
6
|
+
body:
|
|
7
|
+
encoding: UTF-8
|
|
8
|
+
string: '{"model":"google/gemini-3-flash-preview","messages":[{"role":"system","content":"You
|
|
9
|
+
are an evaluation judge. You will be given a response and one or more evaluation
|
|
10
|
+
criteria.\nYour job is to score how well the response meets the criteria.\n\nScoring:\n-
|
|
11
|
+
0: Completely fails to meet the criteria\n- 1-3: Mostly fails, with minor
|
|
12
|
+
positive elements\n- 4-6: Partially meets criteria, significant room for improvement\n-
|
|
13
|
+
7-8: Mostly meets criteria with minor issues\n- 9: Meets criteria well\n-
|
|
14
|
+
10: Perfectly meets all criteria\n\nBe strict but fair. Consider each criterion
|
|
15
|
+
carefully.\n\nYou MUST respond with valid JSON in this exact format:\n{\"score\":
|
|
16
|
+
<0-10>, \"reasoning\": \"Brief explanation of the score\"}\n\nYour reasoning
|
|
17
|
+
should be concise (1-2 sentences max).\n"},{"role":"user","content":"## Response
|
|
18
|
+
to evaluate:\nHello! I''m doing great, thanks for asking. How can I help you
|
|
19
|
+
today?\n\n## Evaluation criteria:\nis a greeting response\n\nScore this response
|
|
20
|
+
from 0-10. Respond with JSON only."}],"response_format":{"type":"json_object"}}'
|
|
21
|
+
headers:
|
|
22
|
+
Content-Type:
|
|
23
|
+
- application/json
|
|
24
|
+
Authorization:
|
|
25
|
+
- Bearer <API_KEY>
|
|
26
|
+
User-Agent:
|
|
27
|
+
- Faraday v2.14.0
|
|
28
|
+
response:
|
|
29
|
+
status:
|
|
30
|
+
code: 200
|
|
31
|
+
message: OK
|
|
32
|
+
headers:
|
|
33
|
+
date:
|
|
34
|
+
- Thu, 25 Dec 2025 01:40:20 GMT
|
|
35
|
+
content-type:
|
|
36
|
+
- application/json
|
|
37
|
+
transfer-encoding:
|
|
38
|
+
- chunked
|
|
39
|
+
connection:
|
|
40
|
+
- keep-alive
|
|
41
|
+
content-encoding:
|
|
42
|
+
- gzip
|
|
43
|
+
access-control-allow-origin:
|
|
44
|
+
- "*"
|
|
45
|
+
vary:
|
|
46
|
+
- Accept-Encoding
|
|
47
|
+
permissions-policy:
|
|
48
|
+
- payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com"
|
|
49
|
+
"https://js.stripe.com" "https://*.js.stripe.com" "https://hooks.stripe.com")
|
|
50
|
+
referrer-policy:
|
|
51
|
+
- no-referrer, strict-origin-when-cross-origin
|
|
52
|
+
x-content-type-options:
|
|
53
|
+
- nosniff
|
|
54
|
+
server:
|
|
55
|
+
- cloudflare
|
|
56
|
+
cf-ray:
|
|
57
|
+
- 9b34a1360819a20d-MSP
|
|
58
|
+
body:
|
|
59
|
+
encoding: ASCII-8BIT
|
|
60
|
+
string: "\n \n{\"id\":\"gen-1766626819-cwszKPb5TYdNlsdM6wXx\",\"provider\":\"Google
|
|
61
|
+
AI Studio\",\"model\":\"google/gemini-3-flash-preview\",\"object\":\"chat.completion\",\"created\":1766626819,\"choices\":[{\"logprobs\":null,\"finish_reason\":\"stop\",\"native_finish_reason\":\"STOP\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"{\\\"score\\\":
|
|
62
|
+
10, \\\"reasoning\\\": \\\"The response is a perfect example of a polite greeting
|
|
63
|
+
that answers the initial check-in and offers further assistance.\\\"}\",\"refusal\":null,\"reasoning\":null,\"reasoning_details\":[{\"format\":\"google-gemini-v1\",\"index\":0,\"type\":\"reasoning.encrypted\",\"data\":\"EjQKMgFyyNp8jSP3SGP1mfnfQ8XosyevKsQG2o0utylFyYWkt17Q92DKVN2vMFNAtAwMN8B8\"}]}}],\"usage\":{\"prompt_tokens\":219,\"completion_tokens\":35,\"total_tokens\":254,\"cost\":0.0002145,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.0001095,\"upstream_inference_completions_cost\":0.000105},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0}}}"
|
|
64
|
+
recorded_at: Thu, 25 Dec 2025 01:40:20 GMT
|
|
65
|
+
recorded_with: VCR 6.4.0
|
data/examples/cassettes/qualspec_rspec_integration_with_context_uses_context_in_evaluation.yml
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
---
|
|
2
|
+
http_interactions:
|
|
3
|
+
- request:
|
|
4
|
+
method: post
|
|
5
|
+
uri: https://openrouter.ai/api/v1/chat/completions
|
|
6
|
+
body:
|
|
7
|
+
encoding: UTF-8
|
|
8
|
+
string: '{"model":"google/gemini-3-flash-preview","messages":[{"role":"system","content":"You
|
|
9
|
+
are an evaluation judge. You will be given a response and one or more evaluation
|
|
10
|
+
criteria.\nYour job is to score how well the response meets the criteria.\n\nScoring:\n-
|
|
11
|
+
0: Completely fails to meet the criteria\n- 1-3: Mostly fails, with minor
|
|
12
|
+
positive elements\n- 4-6: Partially meets criteria, significant room for improvement\n-
|
|
13
|
+
7-8: Mostly meets criteria with minor issues\n- 9: Meets criteria well\n-
|
|
14
|
+
10: Perfectly meets all criteria\n\nBe strict but fair. Consider each criterion
|
|
15
|
+
carefully.\n\nYou MUST respond with valid JSON in this exact format:\n{\"score\":
|
|
16
|
+
<0-10>, \"reasoning\": \"Brief explanation of the score\"}\n\nYour reasoning
|
|
17
|
+
should be concise (1-2 sentences max).\n"},{"role":"user","content":"## Response
|
|
18
|
+
to evaluate:\nI''d be happy to help with weather information! However, I don''t
|
|
19
|
+
have access to real-time weather data. You could check a weather service like
|
|
20
|
+
weather.com or your phone''s weather app for accurate forecasts.\n\n## Additional
|
|
21
|
+
context:\nThis agent does not have access to real-time data\n\n## Evaluation
|
|
22
|
+
criteria:\nappropriately handles the request given limitations\n\nScore this
|
|
23
|
+
response from 0-10. Respond with JSON only."}],"response_format":{"type":"json_object"}}'
|
|
24
|
+
headers:
|
|
25
|
+
Content-Type:
|
|
26
|
+
- application/json
|
|
27
|
+
Authorization:
|
|
28
|
+
- Bearer <API_KEY>
|
|
29
|
+
User-Agent:
|
|
30
|
+
- Faraday v2.14.0
|
|
31
|
+
response:
|
|
32
|
+
status:
|
|
33
|
+
code: 200
|
|
34
|
+
message: OK
|
|
35
|
+
headers:
|
|
36
|
+
date:
|
|
37
|
+
- Thu, 25 Dec 2025 01:40:15 GMT
|
|
38
|
+
content-type:
|
|
39
|
+
- application/json
|
|
40
|
+
transfer-encoding:
|
|
41
|
+
- chunked
|
|
42
|
+
connection:
|
|
43
|
+
- keep-alive
|
|
44
|
+
content-encoding:
|
|
45
|
+
- gzip
|
|
46
|
+
access-control-allow-origin:
|
|
47
|
+
- "*"
|
|
48
|
+
vary:
|
|
49
|
+
- Accept-Encoding
|
|
50
|
+
permissions-policy:
|
|
51
|
+
- payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com"
|
|
52
|
+
"https://js.stripe.com" "https://*.js.stripe.com" "https://hooks.stripe.com")
|
|
53
|
+
referrer-policy:
|
|
54
|
+
- no-referrer, strict-origin-when-cross-origin
|
|
55
|
+
x-content-type-options:
|
|
56
|
+
- nosniff
|
|
57
|
+
server:
|
|
58
|
+
- cloudflare
|
|
59
|
+
cf-ray:
|
|
60
|
+
- 9b34a1184fcea213-MSP
|
|
61
|
+
body:
|
|
62
|
+
encoding: ASCII-8BIT
|
|
63
|
+
string: "\n \n\n \n{\"id\":\"gen-1766626814-I8RLvzbYlGHhrnUNgdWj\",\"provider\":\"Google\",\"model\":\"google/gemini-3-flash-preview\",\"object\":\"chat.completion\",\"created\":1766626814,\"choices\":[{\"logprobs\":null,\"finish_reason\":\"stop\",\"native_finish_reason\":\"STOP\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"{\\\"score\\\":
|
|
64
|
+
10, \\\"reasoning\\\": \\\"The response stays within its limitations by clearly
|
|
65
|
+
stating its lack of real-time data while suggesting helpful alternatives.\\\"}\",\"refusal\":null,\"reasoning\":null}}],\"usage\":{\"prompt_tokens\":265,\"completion_tokens\":34,\"total_tokens\":299,\"cost\":0.0002345,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.0001325,\"upstream_inference_completions_cost\":0.000102},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0}}}"
|
|
66
|
+
recorded_at: Thu, 25 Dec 2025 01:40:15 GMT
|
|
67
|
+
recorded_with: VCR 6.4.0
|
data/examples/cassettes/qualspec_rspec_integration_with_rubrics_evaluates_using_builtin_rubrics.yml
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
---
|
|
2
|
+
http_interactions:
|
|
3
|
+
- request:
|
|
4
|
+
method: post
|
|
5
|
+
uri: https://openrouter.ai/api/v1/chat/completions
|
|
6
|
+
body:
|
|
7
|
+
encoding: UTF-8
|
|
8
|
+
string: '{"model":"google/gemini-3-flash-preview","messages":[{"role":"system","content":"You
|
|
9
|
+
are an evaluation judge. You will be given a response and one or more evaluation
|
|
10
|
+
criteria.\nYour job is to score how well the response meets the criteria.\n\nScoring:\n-
|
|
11
|
+
0: Completely fails to meet the criteria\n- 1-3: Mostly fails, with minor
|
|
12
|
+
positive elements\n- 4-6: Partially meets criteria, significant room for improvement\n-
|
|
13
|
+
7-8: Mostly meets criteria with minor issues\n- 9: Meets criteria well\n-
|
|
14
|
+
10: Perfectly meets all criteria\n\nBe strict but fair. Consider each criterion
|
|
15
|
+
carefully.\n\nYou MUST respond with valid JSON in this exact format:\n{\"score\":
|
|
16
|
+
<0-10>, \"reasoning\": \"Brief explanation of the score\"}\n\nYour reasoning
|
|
17
|
+
should be concise (1-2 sentences max).\n"},{"role":"user","content":"## Response
|
|
18
|
+
to evaluate:\nHello! I''m doing great, thanks for asking. How can I help you
|
|
19
|
+
today?\n\n## Evaluation criteria:\nGets to the point without unnecessary preamble\nAvoids
|
|
20
|
+
repetition and filler phrases\nResponse length is appropriate for the question
|
|
21
|
+
complexity\n\nScore this response from 0-10. Respond with JSON only."}],"response_format":{"type":"json_object"}}'
|
|
22
|
+
headers:
|
|
23
|
+
Content-Type:
|
|
24
|
+
- application/json
|
|
25
|
+
Authorization:
|
|
26
|
+
- Bearer <API_KEY>
|
|
27
|
+
User-Agent:
|
|
28
|
+
- Faraday v2.14.0
|
|
29
|
+
response:
|
|
30
|
+
status:
|
|
31
|
+
code: 200
|
|
32
|
+
message: OK
|
|
33
|
+
headers:
|
|
34
|
+
date:
|
|
35
|
+
- Thu, 25 Dec 2025 01:40:16 GMT
|
|
36
|
+
content-type:
|
|
37
|
+
- application/json
|
|
38
|
+
transfer-encoding:
|
|
39
|
+
- chunked
|
|
40
|
+
connection:
|
|
41
|
+
- keep-alive
|
|
42
|
+
content-encoding:
|
|
43
|
+
- gzip
|
|
44
|
+
access-control-allow-origin:
|
|
45
|
+
- "*"
|
|
46
|
+
vary:
|
|
47
|
+
- Accept-Encoding
|
|
48
|
+
permissions-policy:
|
|
49
|
+
- payment=(self "https://checkout.stripe.com" "https://connect-js.stripe.com"
|
|
50
|
+
"https://js.stripe.com" "https://*.js.stripe.com" "https://hooks.stripe.com")
|
|
51
|
+
referrer-policy:
|
|
52
|
+
- no-referrer, strict-origin-when-cross-origin
|
|
53
|
+
x-content-type-options:
|
|
54
|
+
- nosniff
|
|
55
|
+
server:
|
|
56
|
+
- cloudflare
|
|
57
|
+
cf-ray:
|
|
58
|
+
- 9b34a1200ceaa1e2-MSP
|
|
59
|
+
body:
|
|
60
|
+
encoding: ASCII-8BIT
|
|
61
|
+
string: "\n \n\n \n{\"id\":\"gen-1766626816-f7eSETM1M0krGjSvXPua\",\"provider\":\"Google
|
|
62
|
+
AI Studio\",\"model\":\"google/gemini-3-flash-preview\",\"object\":\"chat.completion\",\"created\":1766626816,\"choices\":[{\"logprobs\":null,\"finish_reason\":\"stop\",\"native_finish_reason\":\"STOP\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"{\\\"score\\\":
|
|
63
|
+
7, \\\"reasoning\\\": \\\"The response includes polite social preamble and
|
|
64
|
+
filler that, while friendly, technically violates the 'no unnecessary preamble'
|
|
65
|
+
criterion for a direct task.\\\"}\",\"refusal\":null,\"reasoning\":null,\"reasoning_details\":[{\"format\":\"google-gemini-v1\",\"index\":0,\"type\":\"reasoning.encrypted\",\"data\":\"EjQKMgFyyNp86sUsL/xIHgi50YKtFxIdNguYZ33wRoZ5us2d3/v0qElC5ziEAw6rvOz2yt/z\"}]}}],\"usage\":{\"prompt_tokens\":238,\"completion_tokens\":39,\"total_tokens\":277,\"cost\":0.000236,\"is_byok\":false,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0,\"video_tokens\":0},\"cost_details\":{\"upstream_inference_cost\":null,\"upstream_inference_prompt_cost\":0.000119,\"upstream_inference_completions_cost\":0.000117},\"completion_tokens_details\":{\"reasoning_tokens\":0,\"image_tokens\":0}}}"
|
|
66
|
+
recorded_at: Thu, 25 Dec 2025 01:40:17 GMT
|
|
67
|
+
recorded_with: VCR 6.4.0
|