ruby_llm-providers-typesafe 0.1.0

data/lib/ruby_llm/typesafe/questions.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Typesafe
+    # Collects the questions for one evaluation. Typesafe.evaluate yields
+    # one to its block. Each question gets an id you choose; its answer
+    # comes back under the same id. Ids are for your code and never reach
+    # the model, so the instructions must carry the full meaning.
+    class Questions
+      # One question: its +type+ (:noul, :choice, or :score), its
+      # +instructions+, and its +criteria+.
+      Question = Struct.new(:type, :instructions, :criteria, keyword_init: true)
+
+      def initialize # :nodoc:
+        @questions = {}
+      end
+
+      # Adds a yes/no question. Its answer is the probability that the
+      # answer is yes. +yes:+ and +no:+ optionally describe what each
+      # answer means.
+      #
+      #   q.noul :urgent, "Does this convey urgency?", yes: "Explicitly time-sensitive"
+      #
+      def noul(id, instructions, yes: nil, no: nil)
+        add id, Question.new(type: :noul, instructions: instructions, criteria: { yes: yes, no: no })
+      end
+
+      # Adds a question that picks one of +options+. Pass a Hash of option
+      # to description, or an Array of options that need no description.
+      # The answer names the option with the same key you passed.
+      #
+      #   q.choice :team, "Which team should handle this?", billing: "Payments", technical: "Bugs"
+      #   q.choice :language, "Which language is this written in?", %i[english spanish other]
+      #
+      def choice(id, instructions, options = nil, **described)
+        options = options.is_a?(Array) ? options.to_h { |option| [option, nil] } : (options || {}).merge(described)
+        raise ArgumentError, "choice #{id.inspect} needs at least two options" if options.size < 2
+
+        add id, Question.new(type: :choice, instructions: instructions, criteria: options)
+      end
+
+      # Adds a question that rates the state on ordered +levels+, lowest
+      # first. Each level describes a concrete situation. The answer is a
+      # probability-weighted position between the first and last level.
+      #
+      #   q.score :frustration, "How frustrated is the customer?", ["Calm", "Frustrated", "Very angry"]
+      #
+      def score(id, instructions, levels)
+        raise ArgumentError, "score #{id.inspect} needs at least two levels" if Array(levels).size < 2
+
+        add id, Question.new(type: :score, instructions: instructions, criteria: Array(levels))
+      end
+
+      # Returns whether no questions have been added.
+      def empty?
+        @questions.empty?
+      end
+
+      # Returns the questions as a Hash of id to Question.
+      def to_h
+        @questions.dup
+      end
+
+      private
+
+      def add(id, question)
+        raise ArgumentError, "question #{id.inspect} is already defined" if @questions.key?(id)
+
+        @questions[id] = question
+        self
+      end
+    end
+  end
+end

data/lib/ruby_llm/typesafe.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'ruby_llm'
+require_relative 'typesafe/questions'
+require_relative 'typesafe/evaluation'
+
+module RubyLLM
+  # Typed judgments from TypeSafe's System One models. Ask yes/no, choice,
+  # and score questions about a state and get probabilities back:
+  #
+  #   evaluation = RubyLLM::Typesafe.evaluate("Help! My payouts have been failing for 3 days.") do |q|
+  #     q.noul :urgent, "Does this convey urgency?"
+  #     q.choice :team, "Which team should handle this?",
+  #              billing: "Payments, invoicing, refunds",
+  #              technical: "Bugs, outages, integrations"
+  #     q.score :frustration, "How frustrated is the customer?", ["Calm", "Frustrated", "Very angry"]
+  #   end
+  #
+  #   evaluation[:urgent].probability  # => 0.92
+  #   evaluation[:team].option         # => :technical
+  #   evaluation[:frustration].level   # => "Very angry"
+  #
+  module Typesafe
+    # The model evaluate uses when none is given.
+    DEFAULT_MODEL = 'jev-latest'
+
+    # Evaluates +state+ against the questions the block adds and returns an
+    # Evaluation. +state+ is a String, or a Hash or Array for structured
+    # data. The block receives a Questions builder. Every question runs in
+    # parallel against the same state and cannot see the other answers.
+    # +context:+ takes a RubyLLM::Context to use its configuration.
+    #
+    #   RubyLLM::Typesafe.evaluate(ticket, model: "jev-1.13.0") { |q| q.noul :spam, "Is this spam?" }
+    #
+    def self.evaluate(state, model: DEFAULT_MODEL, context: nil)
+      raise ArgumentError, 'evaluate requires a block that adds questions' unless block_given?
+
+      questions = Questions.new
+      yield questions
+      raise ArgumentError, 'evaluate requires at least one question' if questions.empty?
+
+      config = context&.config || RubyLLM.config
+      model, provider = Models.resolve(model, provider: :typesafe, config: config)
+      provider.evaluate(state, questions, model: model)
+    end
+  end
+end

data/models.json

@@ -0,0 +1,62 @@
+[
+  {
+    "id": "jev-latest",
+    "name": "jev-latest",
+    "provider": "typesafe",
+    "family": null,
+    "created_at": "2026-09-10 18:38:01 UTC",
+    "context_window": null,
+    "max_output_tokens": null,
+    "knowledge_cutoff": null,
+    "modalities": {
+      "input": [
+        "text"
+      ],
+      "output": [
+        "rerank"
+      ]
+    },
+    "capabilities": [],
+    "pricing": {
+      "text_tokens": {
+        "standard": {
+          "input_per_million": 0.042,
+          "output_per_million": 0
+        }
+      }
+    },
+    "metadata": {
+      "description": "The latest iteration of TypeSafe's System One Model: Jev"
+    }
+  },
+  {
+    "id": "jev-preview",
+    "name": "jev-preview",
+    "provider": "typesafe",
+    "family": null,
+    "created_at": "2026-09-10 18:39:06 UTC",
+    "context_window": null,
+    "max_output_tokens": null,
+    "knowledge_cutoff": null,
+    "modalities": {
+      "input": [
+        "text"
+      ],
+      "output": [
+        "rerank"
+      ]
+    },
+    "capabilities": [],
+    "pricing": {
+      "text_tokens": {
+        "standard": {
+          "input_per_million": 0.042,
+          "output_per_million": 0
+        }
+      }
+    },
+    "metadata": {
+      "description": "A preview version of `jev-latest`: should be better in most ways"
+    }
+  }
+]

data/spec/fixtures/vcr_cassettes/rubyllm_rerank_typesafe_jev_latest_orders_documents_by_relevance.yml

@@ -0,0 +1,49 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: "<TYPESAFE_API_BASE>/v1/systemone"
+    body:
+      encoding: UTF-8
+      string: '{"state":{"query":"What is the capital of the United States?","documents":["Carson
+        City is the capital of Nevada.","Washington, D.C. is the capital of the United
+        States."]},"model":"jev-latest","questions":{"document_0":{"type":"noul","instructions":"Does
+        `documents[0]` help answer `query`?","criteria":{"true":"The document contains
+        information that answers or directly addresses the query.","false":"The document
+        is off topic or only shares words with the query."}},"document_1":{"type":"noul","instructions":"Does
+        `documents[1]` help answer `query`?","criteria":{"true":"The document contains
+        information that answers or directly addresses the query.","false":"The document
+        is off topic or only shares words with the query."}}}}'
+    headers:
+      User-Agent:
+      - Faraday v2.14.4
+      Authorization:
+      - Bearer <AUTH_TOKEN>
+      Content-Type:
+      - application/json
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Fri, 18 Sep 2026 12:23:44 GMT
+      Server:
+      - istio-envoy
+      Content-Length:
+      - '164'
+      Content-Type:
+      - application/json
+      X-Typesafe-Request-Id:
+      - req_01a0b47903987843bd07f5fc60a186b8
+      X-Envoy-Upstream-Service-Time:
+      - '88'
+    body:
+      encoding: UTF-8
+      string: '{"model":"jev-1.13.0","answers":{"document_0":{"type":"noul","noul":0.03},"document_1":{"type":"noul","noul":0.99}},"usage":{"input_tokens":430,"output_tokens":40}}'
+  recorded_at: Fri, 18 Sep 2026 12:23:44 GMT
+recorded_with: VCR 6.4.0

data/spec/fixtures/vcr_cassettes/rubyllm_typesafe_evaluation_typesafe_jev_latest_answers_noul_choice_and_score_questions.yml

@@ -0,0 +1,46 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: "<TYPESAFE_API_BASE>/v1/systemone"
+    body:
+      encoding: UTF-8
+      string: '{"state":"Help! My payouts have been failing for 3 days.","model":"jev-latest","questions":{"urgent":{"type":"noul","instructions":"Does
+        this convey urgency?"},"team":{"type":"choice","instructions":"Which team
+        should handle this?","criteria":{"billing":"Payments, invoicing, refunds","technical":"Bugs,
+        outages, integrations","sales":"Pricing, upgrades, new accounts"}},"frustration":{"type":"score","instructions":"How
+        frustrated is the customer?","criteria":["Calm","Frustrated","Very angry"]}}}'
+    headers:
+      User-Agent:
+      - Faraday v2.14.4
+      Authorization:
+      - Bearer <AUTH_TOKEN>
+      Content-Type:
+      - application/json
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Fri, 18 Sep 2026 12:23:43 GMT
+      Server:
+      - istio-envoy
+      Content-Length:
+      - '405'
+      Content-Type:
+      - application/json
+      X-Typesafe-Request-Id:
+      - req_01a0b479007b7d67ae86981d477509c0
+      X-Envoy-Upstream-Service-Time:
+      - '100'
+    body:
+      encoding: UTF-8
+      string: '{"model":"jev-1.13.0","answers":{"urgent":{"type":"noul","noul":0.95},"team":{"type":"choice","choice":"billing","confidence":0.77,"probabilities":{"technical":0.15,"billing":0.85,"sales":0.0}},"frustration":{"type":"score","score":1.05,"confidence":0.93,"legend":{"0":"Calm","1":"Frustrated","2":"Very
+        angry"},"probabilities":{"0":0.0,"1":0.95,"2":0.05}}},"usage":{"input_tokens":402,"output_tokens":70}}'
+  recorded_at: Fri, 18 Sep 2026 12:23:44 GMT
+recorded_with: VCR 6.4.0

data/spec/ruby_llm/models_spec.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe RubyLLM::Models do
+  include_context 'with configured RubyLLM'
+
+  it 'accepts versioned model ids that the listing leaves out' do
+    expect(RubyLLM::Providers::Typesafe.assume_models_exist?).to be(true)
+  end
+end

data/spec/ruby_llm/providers/typesafe/system_one_spec.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe RubyLLM::Providers::Typesafe::SystemOne do
+  subject(:protocol) { described_class.new(provider, model) }
+
+  let(:config) do
+    RubyLLM::Configuration.new.tap { |provider_config| provider_config.typesafe_api_key = 'test-key' }
+  end
+  let(:provider) { RubyLLM::Providers::Typesafe.new(config) }
+  let(:model) do
+    RubyLLM::Model.new(id: 'jev-latest', provider: 'typesafe',
+                       pricing: { text_tokens: { standard: { input_per_million: 0.042, output_per_million: 0 } } })
+  end
+  let(:response) { Struct.new(:body) }
+
+  def questions(&)
+    RubyLLM::Typesafe::Questions.new.tap(&)
+  end
+
+  describe '#render_evaluation_payload' do
+    it 'renders each question type in the wire vocabulary' do
+      payload = protocol.render_evaluation_payload(
+        { ticket: 'Payouts failing' },
+        questions do |q|
+          q.noul :urgent, 'Does this convey urgency?', yes: 'Time-sensitive', no: 'Not urgent'
+          q.choice :team, 'Which team?', billing: 'Payments', technical: nil
+          q.score :frustration, 'How frustrated?', %w[Calm Angry]
+        end,
+        model: 'jev-latest'
+      )
+
+      expect(payload).to eq(
+        state: { ticket: 'Payouts failing' },
+        model: 'jev-latest',
+        questions: {
+          'urgent' => { type: 'noul', instructions: 'Does this convey urgency?',
+                        criteria: { true => 'Time-sensitive', false => 'Not urgent' } },
+          'team' => { type: 'choice', instructions: 'Which team?',
+                      criteria: { 'billing' => 'Payments', 'technical' => nil } },
+          'frustration' => { type: 'score', instructions: 'How frustrated?', criteria: %w[Calm Angry] }
+        }
+      )
+    end
+
+    it 'omits noul criteria when neither answer is described' do
+      payload = protocol.render_evaluation_payload('text', questions { |q| q.noul :spam, 'Is this spam?' },
+                                                   model: 'jev-latest')
+
+      expect(payload[:questions]['spam']).to eq(type: 'noul', instructions: 'Is this spam?')
+    end
+  end
+
+  describe '#parse_evaluation_response' do
+    let(:asked) do
+      questions do |q|
+        q.noul :urgent, 'Does this convey urgency?'
+        q.choice :team, 'Which team?', %i[billing technical]
+        q.score :frustration, 'How frustrated?', ['Calm', 'Frustrated', 'Very angry']
+      end
+    end
+    let(:body) do
+      {
+        'model' => 'jev-1.13.0',
+        'answers' => {
+          'urgent' => { 'type' => 'noul', 'noul' => 0.92 },
+          'team' => { 'type' => 'choice', 'choice' => 'technical',
+                      'probabilities' => { 'billing' => 0.15, 'technical' => 0.85 }, 'confidence' => 0.82 },
+          'frustration' => { 'type' => 'score', 'score' => 1.6,
+                             'legend' => { '0' => 'Calm', '1' => 'Frustrated', '2' => 'Very angry' },
+                             'probabilities' => { '0' => 0.05, '1' => 0.3, '2' => 0.65 }, 'confidence' => 0.78 }
+        },
+        'usage' => { 'input_tokens' => 312, 'output_tokens' => 48 }
+      }
+    end
+    let(:evaluation) { protocol.parse_evaluation_response(response.new(body), asked) }
+
+    it 'reads a noul answer as a probability' do
+      expect(evaluation[:urgent].probability).to eq(0.92)
+      expect(evaluation[:urgent]).to be_yes
+      expect(evaluation[:urgent].yes?(threshold: 0.95)).to be(false)
+    end
+
+    it 'maps a choice answer back to the option keys that were asked' do
+      expect(evaluation[:team].option).to eq(:technical)
+      expect(evaluation[:team].probabilities).to eq(billing: 0.15, technical: 0.85)
+      expect(evaluation[:team].confidence).to eq(0.82)
+    end
+
+    it 'orders score probabilities by level' do
+      expect(evaluation[:frustration].score).to eq(1.6)
+      expect(evaluation[:frustration].probabilities).to eq([0.05, 0.3, 0.65])
+      expect(evaluation[:frustration].level).to eq('Very angry')
+    end
+
+    it 'reports the answering model, tokens, and input-only cost' do
+      expect(evaluation.model).to eq('jev-1.13.0')
+      expect(evaluation.tokens.input).to eq(312)
+      expect(evaluation.tokens.output).to eq(48)
+      expect(evaluation.cost.total).to be_within(1e-12).of(312 * 0.042 / 1_000_000)
+    end
+  end
+
+  describe 'reranking' do
+    let(:documents) { ['Carson City is the capital of Nevada.', 'Washington, D.C. is the capital of the US.'] }
+    let(:body) do
+      {
+        'model' => 'jev-1.13.0',
+        'answers' => { 'document_0' => { 'type' => 'noul', 'noul' => 0.1 },
+                       'document_1' => { 'type' => 'noul', 'noul' => 0.95 } },
+        'usage' => { 'input_tokens' => 120, 'output_tokens' => 4 }
+      }
+    end
+
+    it 'asks one relevance question per document over a shared state' do
+      payload = protocol.render_rerank_payload('Capital of the US?', documents, model: 'jev-latest')
+
+      expect(payload[:state]).to eq(query: 'Capital of the US?', documents: documents)
+      expect(payload[:questions].keys).to eq(%w[document_0 document_1])
+      expect(payload[:questions]['document_1']).to include(type: 'noul',
+                                                           instructions: 'Does `documents[1]` help answer `query`?')
+    end
+
+    it 'merges provider options into the payload' do
+      payload = protocol.render_rerank_payload('q', documents, model: 'jev-latest',
+                                                               provider_options: { model: 'jev-preview' })
+
+      expect(payload[:model]).to eq('jev-preview')
+    end
+
+    it 'sorts documents by the probability that they are relevant' do
+      protocol.render_rerank_payload('Capital of the US?', documents, model: 'jev-latest')
+      rerank = protocol.parse_rerank_response(response.new(body), model: 'jev-latest', documents: documents)
+
+      expect(rerank.results.map(&:index)).to eq([1, 0])
+      expect(rerank.results.first.score).to eq(0.95)
+      expect(rerank.results.first.document).to eq(documents[1])
+      expect(rerank.model).to eq('jev-1.13.0')
+      expect(rerank.tokens.input).to eq(120)
+    end
+
+    it 'keeps only the top_n results' do
+      protocol.render_rerank_payload('Capital of the US?', documents, model: 'jev-latest', top_n: 1)
+      rerank = protocol.parse_rerank_response(response.new(body), model: 'jev-latest', documents: documents)
+
+      expect(rerank.results.map(&:index)).to eq([1])
+    end
+  end
+
+  describe '#parse_list_models_response' do
+    it 'builds rerank models priced per input token' do
+      body = { 'models' => [{ 'name' => 'jev-latest', 'description' => 'Flagship', 'release_date' => '2026-06-01' }] }
+      models = protocol.parse_list_models_response(response.new(body), 'typesafe')
+
+      expect(models.map(&:id)).to eq(['jev-latest'])
+      expect(models.first.type).to eq(:rerank)
+      expect(models.first.pricing.text_tokens.input).to eq(0.042)
+      expect(models.first.metadata).to eq(description: 'Flagship')
+    end
+  end
+end

data/spec/ruby_llm/providers/typesafe_spec.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe RubyLLM::Providers::Typesafe do
+  subject(:provider) { described_class.new(config) }
+
+  let(:config) do
+    RubyLLM::Configuration.new.tap do |provider_config|
+      provider_config.typesafe_api_key = 'test-key'
+      provider_config.typesafe_api_base = 'https://example.test'
+    end
+  end
+
+  it 'is registered with RubyLLM' do
+    expect(RubyLLM::Provider.resolve(:typesafe)).to eq(described_class)
+  end
+
+  it 'speaks the System One protocol' do
+    expect(described_class.protocols).to eq(system_one: described_class::SystemOne)
+  end
+
+  it 'declares provider configuration' do
+    expect(described_class.configuration_options).to eq(%i[typesafe_api_key typesafe_api_base])
+    expect(described_class.configuration_requirements).to eq(%i[typesafe_api_key])
+  end
+
+  it 'uses configured API base and bearer token' do
+    expect(provider.api_base).to eq('https://example.test')
+    expect(provider.headers).to eq('Authorization' => 'Bearer test-key')
+  end
+
+  it 'defaults to the public API base' do
+    config.typesafe_api_base = nil
+
+    expect(provider.api_base).to eq('https://api.typesafe.ai')
+  end
+
+  it 'reads the message from an error detail' do
+    response = Struct.new(:body).new(
+      { 'detail' => { 'error_type' => 'authentication_error', 'message' => 'Cannot authenticate with the server.' } }
+    )
+
+    expect(provider.parse_error(response)).to eq('Cannot authenticate with the server.')
+  end
+
+  it 'joins validation details into the error message' do
+    response = Struct.new(:body).new({ 'detail' => [{ 'msg' => 'Field required' }, { 'msg' => 'Too short' }] })
+
+    expect(provider.parse_error(response)).to eq('Field required. Too short')
+  end
+
+  it 'falls back to the standard error message' do
+    response = Struct.new(:body).new({ 'error' => { 'message' => 'Invalid API key' } })
+
+    expect(provider.parse_error(response)).to eq('Invalid API key')
+  end
+end

data/spec/ruby_llm/rerank_request_spec.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe RubyLLM::Rerank do
+  include_context 'with configured RubyLLM'
+
+  around do |example|
+    VCR.turned_off { example.run }
+  end
+
+  before do
+    stub_request(:post, 'https://api.typesafe.ai/v1/systemone').to_return(
+      status: 200,
+      headers: { 'Content-Type' => 'application/json' },
+      body: { model: 'jev-1.13.0',
+              answers: { 'document_0' => { type: 'noul', noul: 0.2 }, 'document_1' => { type: 'noul', noul: 0.9 } },
+              usage: { input_tokens: 100, output_tokens: 4 } }.to_json
+    )
+  end
+
+  it 'ranks documents and records usage' do
+    rerank = RubyLLM.rerank('Capital of the US?', ['Carson City is in Nevada.', 'Washington, D.C.'],
+                            model: 'jev-latest', provider: :typesafe)
+
+    expect(rerank.results.map(&:document)).to eq(['Washington, D.C.', 'Carson City is in Nevada.'])
+    expect(rerank.tokens.input).to eq(100)
+  end
+
+  it 'raises a RubyLLM error for an invalid API key' do
+    stub_request(:post, 'https://api.typesafe.ai/v1/systemone')
+      .to_return(status: 401, headers: { 'Content-Type' => 'application/json' },
+                 body: { detail: 'Invalid API key' }.to_json)
+
+    expect { RubyLLM.rerank('q', %w[a b], model: 'jev-latest', provider: :typesafe) }
+      .to raise_error(RubyLLM::UnauthorizedError, 'Invalid API key')
+  end
+end

data/spec/ruby_llm/rerank_spec.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe RubyLLM::Rerank, :live do
+  include_context 'with configured RubyLLM'
+
+  each_model(RERANK_MODELS) do |provider, model|
+    it "#{provider}/#{model} orders documents by relevance" do
+      rerank = RubyLLM.rerank(
+        'What is the capital of the United States?',
+        ['Carson City is the capital of Nevada.', 'Washington, D.C. is the capital of the United States.'],
+        model: model,
+        provider: provider,
+        assume_model_exists: true
+      )
+
+      expect(rerank.results.first.document).to include('Washington')
+      expect(rerank.results.first.score).to be > rerank.results.last.score
+      expect(rerank.results.map(&:index)).to contain_exactly(0, 1)
+    end
+  end
+end

data/spec/ruby_llm/typesafe/evaluation_spec.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe RubyLLM::Typesafe::Evaluation, :live do
+  include_context 'with configured RubyLLM'
+
+  each_model(EVALUATION_MODELS) do |provider, model|
+    it "#{provider}/#{model} answers noul, choice, and score questions" do
+      evaluation = RubyLLM::Typesafe.evaluate('Help! My payouts have been failing for 3 days.', model: model) do |q|
+        q.noul :urgent, 'Does this convey urgency?'
+        q.choice :team, 'Which team should handle this?',
+                 billing: 'Payments, invoicing, refunds',
+                 technical: 'Bugs, outages, integrations',
+                 sales: 'Pricing, upgrades, new accounts'
+        q.score :frustration, 'How frustrated is the customer?', ['Calm', 'Frustrated', 'Very angry']
+      end
+
+      expect(evaluation[:urgent].probability).to be_between(0, 1)
+      expect(evaluation[:team].probabilities.keys).to contain_exactly(:billing, :technical, :sales)
+      expect(evaluation[:team].probabilities.values.sum).to be_within(0.01).of(1)
+      expect(evaluation[:frustration].score).to be_between(0, 2)
+      expect(evaluation[:frustration].probabilities.size).to eq(3)
+      expect(evaluation.tokens.input).to be_positive
+    end
+  end
+end

data/spec/ruby_llm/typesafe_spec.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe RubyLLM::Typesafe do
+  include_context 'with configured RubyLLM'
+
+  describe '.evaluate' do
+    before do
+      stub_request(:post, 'https://api.typesafe.ai/v1/systemone')
+        .with(body: hash_including('model' => 'jev-latest', 'state' => 'Help! My payouts have been failing.'))
+        .to_return(
+          status: 200,
+          headers: { 'Content-Type' => 'application/json' },
+          body: { model: 'jev-1.13.0', answers: { urgent: { type: 'noul', noul: 0.9 } },
+                  usage: { input_tokens: 20, output_tokens: 2 } }.to_json
+        )
+    end
+
+    around do |example|
+      VCR.turned_off { example.run }
+    end
+
+    it 'returns an Evaluation keyed by question id' do
+      evaluation = described_class.evaluate('Help! My payouts have been failing.') do |q|
+        q.noul :urgent, 'Does this convey urgency?'
+      end
+
+      expect(evaluation).to be_a(RubyLLM::Typesafe::Evaluation)
+      expect(evaluation[:urgent].probability).to eq(0.9)
+      expect(evaluation.model).to eq('jev-1.13.0')
+    end
+
+    it 'requires a block' do
+      expect { described_class.evaluate('text') }.to raise_error(ArgumentError, /requires a block/)
+    end
+
+    it 'requires at least one question' do
+      expect { described_class.evaluate('text') { nil } }.to raise_error(ArgumentError, /at least one question/)
+    end
+  end
+
+  describe 'questions' do
+    subject(:questions) { RubyLLM::Typesafe::Questions.new }
+
+    it 'rejects a duplicate id' do
+      questions.noul :spam, 'Is this spam?'
+
+      expect { questions.noul :spam, 'Is this spam?' }.to raise_error(ArgumentError, /already defined/)
+    end
+
+    it 'requires two options for a choice' do
+      expect { questions.choice :team, 'Which team?', billing: 'Payments' }.to raise_error(ArgumentError)
+    end
+
+    it 'requires two levels for a score' do
+      expect { questions.score :tone, 'How warm?', ['Cold'] }.to raise_error(ArgumentError)
+    end
+  end
+end