llm-evals 0.4.0 → 0.5.0

data/lib/llm/evals/models/throttle.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+require "limiter"
+
+module LLM
+  module Evals
+    module Models
+      # Every model call passes through here, in one process, whatever the -j and
+      # -p. A call waits for each limit that matches it, narrowest first, so a
+      # slot on the shared limit is not spent while the model's own is still shut.
+      # Limits are keyed "openrouter", "openrouter/anthropic" and
+      # "openrouter/anthropic/claude-fable-5".
+      #
+      # RubyLLM's retries are off, since they neither pass through the limits nor
+      # tell other threads. A call the provider says may succeed later is retried
+      # here: it shuts its vendor for every thread, for as long as `Retry-After`
+      # says, or for a growing delay when it says nothing. The number of attempts
+      # is bounded, and so is the wait, so that a run ends with errors and not at all.
+      class Throttle
+        # One `rate_limit`: the first `burst` calls of the run go at once, then
+        # `per_minute`, evenly spaced. A burst is spent once and never refills, so
+        # this is a head start and not a token bucket.
+        class Limit
+          # @param burst [Integer] calls let through at once, before any spacing
+          # @param per_minute [Integer] the evenly spaced rate after that
+          def initialize(burst:, per_minute:)
+            @burst = burst
+            @queue = Limiter::RateQueue.new(per_minute, interval: 60, balanced: true)
+            @lock = Mutex.new
+          end
+
+          # Blocks until this call may go.
+          #
+          # @return [void]
+          def acquire
+            free = @lock.synchronize { (@burst -= 1) >= 0 }
+            @queue.shift unless free
+          end
+        end
+
+        # Tries of one call, the first included, before its error stands.
+        MAX_ATTEMPTS = 5
+        # The longest growing delay, in seconds, when a provider asks for none.
+        MAX_BACKOFF = 30
+
+        # A provider asking for longer than this has really said no.
+        MAX_PAUSE = 300
+
+        # The fiber-local key holding the seconds this thread has waited here.
+        WAITED = :llm_evals_throttle_waited
+
+        # Runs the block and says how long it spent waiting here, so that latency
+        # measures the model and not this lineup's limits. Waits are counted per
+        # thread, so those of threads an adapter starts itself are not seen.
+        #
+        # @yieldreturn [Object]
+        # @return [Array(Object, Float)] the block's value, and the seconds waited on this thread
+        def self.measure
+          before = Thread.current[WAITED].to_f
+          value = yield
+          [value, Thread.current[WAITED].to_f - before]
+        end
+
+        # @param limits [Hash{String => Limit}] by scope
+        # @param sleeper [#call] takes seconds
+        def initialize(limits = {}, sleeper: method(:sleep))
+          @limits = limits
+          @sleeper = sleeper
+          @paused_until = Hash.new(0.0)
+          @lock = Mutex.new
+        end
+
+        # @param model [Model]
+        # @yieldreturn [Object] the call itself, and nothing else: a retry runs the
+        #   block again, so it must be safe to repeat
+        # @return [Object] the block's value
+        # @raise [StandardError] what the call raised, when it cannot or may not be retried
+        def call(model)
+          vendor = vendor_scope(model)
+          attempts = 1
+          begin
+            waiting { admit(model, vendor) }
+            yield
+          rescue StandardError => e
+            delay = delay(model.provider, e, attempts)
+            raise if delay.nil?
+
+            pause(vendor, delay)
+            attempts += 1
+            retry
+          end
+        end
+
+        private
+
+        # Waits out the vendor's pause, then every limit, narrowest first.
+        #
+        # @param model [Model] the model being called
+        # @param vendor [String] the scope a pause would shut
+        # @return [void]
+        def admit(model, vendor)
+          hold(vendor)
+          scopes(model).reverse_each { @limits[it]&.acquire }
+        end
+
+        # @param model [Model] the model being called
+        # @return [Array<String>] "p", "p/vendor", "p/vendor/model"; "p", "p/model" for a provider with no vendors
+        def scopes(model)
+          parts = model.name.split("/")
+          (0..parts.size).map { [model.provider.name, *parts.first(it)].join("/") }
+        end
+
+        # What a retryable failure shuts: the vendor, or the whole provider when it has no vendors.
+        #
+        # @param model [Model] the model being called
+        # @return [String] the scope
+        def vendor_scope(model) = scopes(model)[model.name.include?("/") ? 1 : 0]
+
+        # @param provider [Providers::Base] says whether the error is retryable, and how long it asked
+        # @param error [StandardError] what the call raised
+        # @param attempts [Integer] tries so far, this one included
+        # @return [Numeric, nil] seconds to wait before another attempt; nil to give up
+        def delay(provider, error, attempts)
+          return unless attempts < MAX_ATTEMPTS && provider.retryable?(error)
+
+          asked = provider.retry_after(error)
+          return [2**(attempts - 1), MAX_BACKOFF].min if asked.nil?
+
+          asked unless asked > MAX_PAUSE
+        end
+
+        # Shuts a scope for every thread, never shortening a pause already in force.
+        #
+        # @param scope [String] the scope to shut
+        # @param seconds [Numeric] how long from now
+        # @return [void]
+        def pause(scope, seconds)
+          @lock.synchronize { @paused_until[scope] = [@paused_until[scope], now + seconds].max }
+        end
+
+        # Sleeps until a scope is open, checking again in case another thread extended the pause.
+        #
+        # @param scope [String] the scope to wait on
+        # @return [void]
+        def hold(scope)
+          while (left = @lock.synchronize { @paused_until[scope] } - now).positive?
+            @sleeper.call(left)
+          end
+        end
+
+        # Adds the time the block takes to this thread's total, even when it raises.
+        #
+        # @yield the wait to measure
+        # @yieldreturn [Object]
+        # @return [Object] the block's value
+        def waiting
+          started = now
+          yield
+        ensure
+          Thread.current[WAITED] = Thread.current[WAITED].to_f + (now - started)
+        end
+
+        # A clock that wall-clock changes cannot move.
+        #
+        # @return [Float] monotonic seconds
+        def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/llm/evals/models.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # What is being evaluated: a model at one setting, and how calls to it are limited.
+    module Models
+    end
+  end
+end

data/lib/llm/evals/providers/base.rb

@@ -3,7 +3,7 @@
 module LLM
   module Evals
     module Providers
-      # What answers a {Model}'s calls: one API, the ids it serves, the keys it
+      # What answers a {Models::Model}'s calls: one API, the ids it serves, the keys it
       # needs and the models it lists. A suite with its own transport
       # subclasses this and registers it through `c.provider`.
       #
@@ -16,6 +16,7 @@ module LLM
         # A suite admits a provider only when its config lists this type.
         #
         # @return [Symbol]
+        # @raise [NotImplementedError] unless a subclass overrides it
         def request_type
           raise NotImplementedError, "#{self.class}#request_type must be one of #{Config::REQUEST_TYPES.join(", ")}"
         end
@@ -27,29 +28,42 @@ module LLM
         def request_type_description = ""
 
         # A direct provider serves a vendor's own API and is reached only
-        # through a `direct:` entry that names it; it never claims an id from
-        # `vendors:`, however the id is spelled.
+        # through its own section of the lineup; it never claims another's id,
+        # however the id is spelled.
         #
         # @return [Boolean]
         def direct? = false
 
+        # Dasherized from the class name; a subclass overrides it when the brand is
+        # spelled differently.
+        #
         # @return [String] short name, for messages and the leaderboard
         def name = self.class.name.split("::").last.gsub(/([a-z\d])([A-Z])/, '\1-\2').downcase
 
+        # Matches on {#prefix}, so most subclasses override that instead of this.
+        #
         # @param model_id [String] as the lineup names it
         # @return [Boolean] whether this provider serves the model
         def serves?(model_id) = model_id.start_with?(prefix)
 
+        # Overridden by a subclass whose ids share a namespace.
+        #
         # @return [String] the id prefix this provider claims; "" claims everything
         def prefix = ""
 
+        # Strips {#prefix}, so the lineup's namespacing never reaches the API.
+        #
         # @param model_id [String]
         # @return [String] the id as the provider's API knows it
         def model_name(model_id) = model_id.delete_prefix(prefix)
 
+        # {Registry#keys_for} collects these across the lineup.
+        #
         # @return [Array<String>] environment variables a run through this provider needs
         def env_keys = []
 
+        # {EFFORTS} unless a subclass narrows or empties it.
+        #
         # @return [Array<String>] the efforts the `-e` flag accepts for this provider's models
         def efforts = EFFORTS
 
@@ -62,27 +76,58 @@ module LLM
 
         # Prepares the client once per process, after the env files loaded.
         #
-        # @option options [String] :app_name what the client tells the API the caller is
-        # @return [void]
+        # @overload setup!(app_name:)
+        #   @param app_name [String] what the client tells the API the caller is
+        #   @return [void]
         def setup!(**) = nil
 
-        # @param model [Model]
-        # @param system [String, nil]
-        # @param user [String]
-        # @param json [Boolean]
-        # @return [Completion]
+        # One chat completion, for a provider of request type `prompt`.
+        #
+        # @param model [Models::Model] the model and settings to call
+        # @param system [String, nil] the system prompt, if any
+        # @param user [String] the user prompt
+        # @param json [Boolean] whether to ask for a JSON object reply
+        # @return [Results::ProviderResponse]
+        # @raise [NotImplementedError] unless a subclass overrides it
         def complete(model, system:, user:, json: true)
-          raise NotImplementedError, "#{self.class}#complete must return an LLM::Evals::Completion"
+          raise NotImplementedError, "#{self.class}#complete must return an LLM::Evals::Results::ProviderResponse"
         end
 
-        # @param contender [Contender]
-        # @return [Model]
-        def model_for(contender) = Model.new(id: contender.model, provider: self, settings: contender.settings)
+        # One request of typed questions, for a provider whose API answers them.
+        #
+        # @param model [Models::Model] the model to ask
+        # @param state [Object] what the questions are about; any JSON-encodable value
+        # @param questions [Hash{String => Hash}] by name
+        # @return [Results::ProviderResponse]
+        # @raise [NotImplementedError] unless a subclass overrides it
+        def ask(model, state:, questions:)
+          raise NotImplementedError, "#{self.class} does not answer typed questions"
+        end
+
+        # {Models::Throttle} asks this after a failed call; false by default, so a
+        # provider that says nothing is never retried.
+        #
+        # @param _error [StandardError] what a call raised
+        # @return [Boolean] whether the same call may work later
+        def retryable?(_error) = false
+
+        # {Models::Throttle} uses this instead of its own backoff when it is not nil.
+        #
+        # @param _error [StandardError] what a call raised
+        # @return [Numeric, nil] the seconds the API asked to wait, if it did
+        def retry_after(_error) = nil
+
+        # Builds the callable model for a lineup entry, on this provider.
+        #
+        # @param contender [Models::Contender] the lineup entry
+        # @param throttle [Models::Throttle] what the model's calls pass through
+        # @return [Models::Model]
+        def model_for(contender, throttle: Models::Throttle.new) = Models::Model.new(id: contender.model, provider: self, settings: contender.settings, throttle:)
 
         # Every id in `model_ids` the provider does not list.
         #
-        # @param model_ids [Array<String>]
-        # @return [Array<String>]
+        # @param model_ids [Array<String>] ids as the lineup names them
+        # @return [Array<String>] none when the provider cannot list its models
         def unknown(model_ids)
           listed = catalog
           listed.nil? ? [] : model_ids - listed

data/lib/llm/evals/providers/catalog.rb

@@ -11,8 +11,11 @@ module LLM
       module Catalog
         module_function
 
-        # @param url [String]
-        # @param headers [Hash{String => String}]
+        # Performs the GET. A status other than 2xx, a socket failure, a timeout and
+        # an unparsable body all come out as the same {Error}.
+        #
+        # @param url [String] the list's address
+        # @param headers [Hash{String => String}] request headers, such as credentials
         # @return [Hash] the parsed JSON body
         # @raise [Error] when the list cannot be read
         def fetch(url, headers = {})

data/lib/llm/evals/providers/open_router.rb

@@ -7,39 +7,86 @@ module LLM
       # traces RubyLLM calls (tokens, latency, cost) and not raw HTTP. Claims
       # every id no other provider does, so it is always asked last.
       class OpenRouter < Base
+        # The model list, which {#catalog} reads.
         URL = "https://openrouter.ai/api/v1/models"
+        # The environment variable holding the API key.
         KEY = "OPENROUTER_API_KEY"
 
         # OpenRouter's reasoning.effort values.
         EFFORTS = %w[none minimal low medium high xhigh max].freeze
 
+        # Overrides {Base#name}, whose dasherizing would say "open-router".
+        #
+        # @return [String] "openrouter"
         def name = "openrouter"
+        # Overrides {Base#request_type}: chat completions.
+        #
+        # @return [Symbol] `:prompt`
         def request_type = :prompt
+        # Overrides {Base#request_type_description}.
+        #
+        # @return [String]
         def request_type_description = "a chat completion: system and user text in, text out"
+        # Overrides {Base#env_keys}.
+        #
+        # @return [Array<String>] just {KEY}
         def env_keys = [KEY]
+        # Overrides {Base#efforts} with {EFFORTS}, OpenRouter's own list.
+        #
+        # @return [Array<String>]
         def efforts = EFFORTS
 
+        # Reads the list without credentials.
+        #
         # @return [Array<String>] every id OpenRouter lists
+        # @raise [Error] when the list cannot be read
+        # @raise [KeyError] when the body has no `data`, or an entry no `id`
         def catalog = Catalog.fetch(URL).fetch("data").map { it.fetch("id") }
 
         # Points RubyLLM at OpenRouter. Requires it here, not at load, so a
         # suite that never reaches OpenRouter never needs the gem.
         #
-        # @param app_name [String]
+        # @param app_name [String] what RubyLLM tells OpenRouter the caller is
         # @return [void]
+        # @raise [MissingDependency] when the ruby_llm gem is absent
         def setup!(app_name:)
           LLM::Evals.optional!("ruby_llm", needed_for: "calling models through OpenRouter")
           RubyLLM.configure do |config|
             config.openrouter_api_key = ENV.fetch(KEY, nil)
             config.openrouter_app_name = app_name
+            config.max_retries = 0 # {Throttle} retries, in step with the limits
+          end
+        end
+
+        # Only classifies the errors RubyLLM and Faraday raise; anything else is final.
+        #
+        # @param error [StandardError] what a call raised
+        # @return [Boolean] rate limits, overloads and dropped connections
+        def retryable?(error)
+          case error
+          when RubyLLM::RateLimitError, RubyLLM::ServerError, RubyLLM::ServiceUnavailableError, RubyLLM::OverloadedError,
+               Faraday::TimeoutError, Faraday::ConnectionFailed then true
+          else false
           end
         end
 
-        # @param model [Model]
-        # @param system [String, nil]
-        # @param user [String]
+        # OpenRouter sends the header only when every upstream hinted at it.
+        #
+        # @param error [StandardError] what a call raised
+        # @return [Numeric, nil] the seconds to wait, when the error carries a usable header
+        def retry_after(error)
+          value = error.respond_to?(:response) && error.response&.response_headers&.[]("retry-after")
+          Float(value, exception: false) if value
+        end
+
+        # Sends one chat through RubyLLM, passing on only the settings the model has.
+        #
+        # @param model [Models::Model] the model and settings to call
+        # @param system [String, nil] the system prompt, if any
+        # @param user [String] the user prompt
         # @param json [Boolean] asks for `response_format: json_object`
-        # @return [Completion] text, plus the tokens and the cost OpenRouter billed
+        # @return [Results::ProviderResponse] text, plus the tokens and the cost OpenRouter billed
+        # @raise [MissingDependency] when the ruby_llm gem is absent
         def complete(model, system:, user:, json: true)
           LLM::Evals.optional!("ruby_llm", needed_for: "calling models through OpenRouter")
           chat = RubyLLM.chat(model: model.name, provider: :openrouter, assume_model_exists: true)
@@ -50,9 +97,9 @@ module LLM
           chat.with_provider_options(response_format: { type: "json_object" }) if json
 
           message = chat.ask(user)
-          Completion.new(
+          Results::ProviderResponse.new(
             text:  message.content.to_s,
-            usage: Usage.new(input_tokens: message.tokens.input.to_i,
+            usage: Stats::Usage.new(input_tokens: message.tokens.input.to_i,
               output_tokens: message.tokens.output.to_i,
               cost_usd: message.tokens.reported_cost.to_f),
             raw:   message

data/lib/llm/evals/providers/registry.rb

@@ -3,17 +3,21 @@
 module LLM
   module Evals
     module Providers
-      # The providers in use. A `vendors:` contender goes to the first
+      # The providers in use. A contender outside a direct provider's section goes to the first
       # provider that serves its id, asked last-registered first, so a
       # suite's own transport beats the defaults and OpenRouter, which claims
-      # everything, is always last. A `direct:` contender goes to the
+      # everything, is always last. A direct contender goes to the
       # provider it names, and a direct provider is never asked about an id.
       class Registry
         include Enumerable
 
+        # OpenRouter comes first in the list because the last is asked first.
+        #
         # @return [Registry] TypeSafe, then OpenRouter for everything else
         def self.default = new([OpenRouter.new, TypeSafe.new])
 
+        # The array is copied, so registering later does not touch the caller's.
+        #
         # @param providers [Array<Base>] the last one is asked first
         def initialize(providers = [])
           @providers = providers.dup
@@ -23,7 +27,7 @@ module LLM
         # helper and a required file that both register the same provider
         # leave one, not two that would be asked twice.
         #
-        # @param provider [Base]
+        # @param provider [Base] the provider to add
         # @return [Base] the provider
         def register(provider)
           @providers.reject! { it.name == provider.name }
@@ -31,20 +35,29 @@ module LLM
           provider
         end
 
+        # Walks the providers in asking order, last-registered first, which is what
+        # every Enumerable lookup in this class relies on.
+        #
+        # @yieldparam provider [Base] each provider
+        # @return [Array<Base>, Enumerator] the reversed list; an Enumerator without a block
         def each(&) = @providers.reverse.each(&)
 
-        # @param target [Contender, String] a contender, or a `vendors:` model id
+        # Picks the provider for a lineup entry or a bare id, by the rules above.
+        #
+        # @param target [Models::Contender, String] a contender, or a model id
         # @return [Base]
         # @raise [Error] when no provider serves it
         def for(target)
-          return direct(target.provider) if target.is_a?(Contender) && target.direct?
+          return direct(target.provider) if target.is_a?(Models::Contender) && target.direct?
 
-          model_id = target.is_a?(Contender) ? target.model : target
+          model_id = target.is_a?(Models::Contender) ? target.model : target
           find { !it.direct? && it.serves?(model_id) } ||
             raise(Error, "no provider serves #{model_id.inspect}; have #{reject(&:direct?).map(&:name).join(", ")}")
         end
 
-        # @param name [Symbol, String] a key under `direct:`
+        # Finds a direct provider by its lineup section name.
+        #
+        # @param name [Symbol, String] a direct provider's section
         # @return [Base]
         # @raise [Error] when no direct provider has the name
         def direct(name)
@@ -52,23 +65,40 @@ module LLM
             raise(Error, "no direct provider #{name.to_s.inspect}; have #{direct_names.join(", ")}")
         end
 
-        # @return [Array<String>] the names `direct:` may use
+        # For {#direct}'s error message.
+        #
+        # @return [Array<String>] the direct providers' names
         def direct_names = select(&:direct?).map(&:name)
 
-        # @param contenders [Array<Contender>]
+        # Deduplicated, so each provider's setup runs once.
+        #
+        # @param contenders [Array<Models::Contender>] the lineup
         # @return [Array<Base>] each provider the contenders use, once
         def used_by(contenders) = contenders.map { self.for(it) }.uniq
 
-        # @param contenders [Array<Contender>]
+        # Deduplicated across providers that share a key.
+        #
+        # @param contenders [Array<Models::Contender>] the lineup
         # @return [Array<String>] the API keys the contenders' providers need
         def keys_for(contenders) = used_by(contenders).flat_map(&:env_keys).uniq
 
-        # @param contender [Contender]
-        # @return [Model]
-        def model_for(contender) = self.for(contender).model_for(contender)
+        # Routes the contender with {#for}, then lets its provider build the model.
+        #
+        # @param contender [Models::Contender] the lineup entry
+        # @param throttle [Models::Throttle] what the model's calls pass through
+        # @return [Models::Model]
+        def model_for(contender, throttle: Models::Throttle.new) = self.for(contender).model_for(contender, throttle:)
 
         # One refused contender: who, and why.
+        #
+        # @!attribute [r] contender
+        #   @return [Models::Contender] the contender left out of the run
+        # @!attribute [r] reason
+        #   @return [String] the request type it needs, against what the suite produces
         Refusal = Data.define(:contender, :reason) do
+          # The line a warning prints.
+          #
+          # @return [String]
           def to_s = "#{contender.model} refused: #{reason}"
         end
 
@@ -77,9 +107,10 @@ module LLM
         # is named, never run, and never scored 0: a model the suite cannot
         # even ask is not a model that answered wrongly.
         #
-        # @param contenders [Array<Contender>]
+        # @param contenders [Array<Models::Contender>] the lineup
         # @param request_types [Array<Symbol>] what the suite's config declares
-        # @return [Array(Array<Contender>, Array<Refusal>)] admitted, refused
+        # @return [Array(Array<Models::Contender>, Array<Refusal>)] admitted, refused
+        # @raise [Error] when no provider serves a contender
         def admit(contenders, request_types:)
           admitted, refused = contenders.partition { request_types.include?(self.for(it).request_type) }
           refusals = refused.map do |contender|
@@ -89,13 +120,15 @@ module LLM
           [admitted, refusals]
         end
 
+        # Providers disagree on efforts, so this is the union of them.
+        #
         # @return [Array<String>] every effort any provider accepts, for the flag's help
         def efforts = flat_map(&:efforts).uniq
 
         # Asks each provider in use for its catalog, and names every model it
         # does not list.
         #
-        # @param contenders [Array<Contender>]
+        # @param contenders [Array<Models::Contender>] the lineup
         # @return [void]
         # @raise [Error] naming every unknown model
         def verify!(contenders)

data/lib/llm/evals/providers/type_safe/client.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+
+module LLM
+  module Evals
+    module Providers
+      class TypeSafe < Base
+        # The HTTP side of typed questions: credentials, transport, and turning
+        # whatever came back into a parsed body or a gem error.
+        class Client
+          # Where typed questions are posted.
+          URL = URI("https://api.typesafe.ai/v1/systemone")
+          # Seconds to wait for a connection.
+          OPEN_TIMEOUT = 10
+          # Seconds to wait for a reply.
+          READ_TIMEOUT = 120
+
+          # Sends one request, reading the key on each call rather than at load.
+          #
+          # @param payload [Hash{Symbol => Object}] `model:`, `state:` and `questions:`
+          # @return [Hash{String => Object}] the reply, which has `answers`
+          # @raise [Error] on a missing key, or a 2xx that is not a usable answer
+          # @raise [HTTPError] on any status but 2xx
+          def ask(payload)
+            key = ENV.fetch(KEY, "")
+            raise Error, "#{KEY} is not set" if key.empty?
+
+            response = post(payload, key)
+            raise HTTPError, response unless response.is_a?(Net::HTTPSuccess)
+
+            parsed(response.body, payload[:model])
+          end
+
+          private
+
+          # Performs the HTTP call; transport errors pass through for {TypeSafe#retryable?} to judge.
+          #
+          # @param payload [Hash{Symbol => Object}] the JSON body
+          # @param key [String] the API key
+          # @return [Net::HTTPResponse]
+          def post(payload, key)
+            request = Net::HTTP::Post.new(URL, "Authorization" => "Bearer #{key}", "Content-Type" => "application/json")
+            request.body = JSON.generate(payload)
+            Net::HTTP.start(URL.host, URL.port, use_ssl: true, open_timeout: OPEN_TIMEOUT, read_timeout: READ_TIMEOUT) { it.request(request) }
+          end
+
+          # Accepts only a JSON object whose `answers` is an object.
+          #
+          # @param text [String] the response body
+          # @param model [String] the model name, for the error message
+          # @return [Hash{String => Object}] the parsed reply
+          # @raise [Error] when the body is not JSON or has no answers
+          def parsed(text, model)
+            body = JSON.parse(text)
+            return body if body.is_a?(Hash) && body["answers"].is_a?(Hash)
+
+            raise Error, "#{model}: unexpected TypeSafe answer, no answers: #{text.to_s[0, 500]}"
+          rescue JSON::ParserError
+            raise Error, "#{model}: unexpected TypeSafe answer, not JSON: #{text.to_s[0, 500]}"
+          end
+        end
+      end
+    end
+  end
+end