agent-evals 0.1.1

data/lib/agent/evals/launcher.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # Starts the CLI. The executable and the specs both build one, each with
+    # its own argv and streams, and it exits only through kernel.exit.
+    class Launcher
+      def initialize(argv, stdin = $stdin, stdout = $stdout, stderr = $stderr, kernel = Kernel)
+        @argv = argv
+        @stdin = stdin
+        @stdout = stdout
+        @stderr = stderr
+        @kernel = kernel
+      end
+
+      def execute!
+        @kernel.exit(run)
+      end
+
+      private
+
+      # dry-cli exits by itself after help or a usage error; its status is kept.
+      def run
+        argv, helper = Helper.extract(@argv)
+        Helper.load!(Helper.locate(helper))
+        Dry::CLI.new(CLI).call(arguments: argv, out: @stdout, err: @stderr)
+        0
+      rescue SystemExit => e
+        e.status
+      rescue StandardError => e
+        @stderr.puts "error: #{e.message}"
+        @stderr.puts(e.backtrace) if @argv.intersect?(%w[-v --verbose])
+        1
+      end
+    end
+  end
+end

data/lib/agent/evals/metric.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # One number a {Scorer} reports per case. `aggregate` is how the run
+    # combines them (only :mean today), skipping nil, so a metric a case does
+    # not apply to never drags its mean down with a free 0 or up with a free 1.
+    # The `primary` metric ranks the leaderboard and picks the best value.
+    class Metric < Data.define(:name, :description, :aggregate, :primary)
+      AGGREGATES = %i[mean].freeze
+
+      def initialize(name:, description: "", aggregate: :mean, primary: false)
+        raise ArgumentError, "aggregate must be one of #{AGGREGATES.join(", ")}" unless AGGREGATES.include?(aggregate)
+
+        super(name: name.to_s, description: description, aggregate: aggregate, primary: primary)
+      end
+
+      # @param values [Array<Float, nil>] one per case
+      # @return [Float, nil] nil when no case reported the metric
+      def combine(values)
+        present = values.compact
+        present.sum.fdiv(present.size) unless present.empty?
+      end
+    end
+  end
+end

data/lib/agent/evals/model.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # One model at one setting, reached through its provider. Adapters call
+    # {#complete}; the settings an experiment is named by are read here.
+    class Model
+      # The settings every provider understands; a provider may accept more.
+      SETTINGS = %i[effort temperature max_tokens].freeze
+
+      attr_reader :id, :provider, :settings
+
+      # @param id [String] the model id as the lineup names it, e.g. "anthropic/claude-haiku-5.5"
+      # @param provider [Providers::Base] what answers {#complete}
+      # @param settings [Hash{Symbol => Object}] effort, temperature, max_tokens; nil leaves the model's default
+      def initialize(id:, provider:, settings: {})
+        @id = id
+        @provider = provider
+        @settings = settings.transform_keys(&:to_sym).freeze
+      end
+
+      # @return [String, nil] reasoning effort; nil sends none
+      def effort = settings[:effort]
+
+      # @return [Float, nil] sampling temperature; nil sends none
+      def temperature = settings[:temperature]
+
+      # @return [Integer, nil] output token cap
+      def max_tokens = settings[:max_tokens]
+
+      # @return [String] the id without the provider's prefix, as the provider's API knows it
+      def name = provider.model_name(id)
+
+      # @return [String] "provider:id", unique where two providers render the same id
+      def key = "#{provider.name}:#{id}"
+
+      # One call. Transport failures raise; they are the run's errors, not the model's.
+      #
+      # @param system [String, nil]
+      # @param user [String]
+      # @param json [Boolean] ask the provider for a JSON object reply where it supports that
+      # @return [Completion]
+      def complete(user:, system: nil, json: true)
+        provider.complete(self, system: system, user: user, json: json)
+      end
+
+      # @return [String] e.g. "effort=low t=0.2"
+      def label
+        "effort=#{effort || "default"} t=#{temperature.nil? ? "default" : temperature}"
+      end
+
+      def to_s = "#{id} #{label}"
+    end
+  end
+end

data/lib/agent/evals/providers/base.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    module Providers
+      # What answers a {Model}'s calls: one API, the ids it serves, the keys it
+      # needs and the models it lists. A suite with its own transport
+      # subclasses this and registers it through `c.provider`.
+      #
+      # @abstract Override {#serves?}, {#env_keys}, {#catalog} and {#complete}.
+      class Base
+        # The reasoning efforts this provider's API accepts.
+        EFFORTS = %w[none minimal low medium high xhigh max].freeze
+
+        # The kind of request this provider answers; see {Config::REQUEST_TYPES}.
+        # A suite admits a provider only when its config lists this type.
+        #
+        # @return [Symbol]
+        def request_type
+          raise NotImplementedError, "#{self.class}#request_type must be one of #{Config::REQUEST_TYPES.join(", ")}"
+        end
+
+        # One line on what the request type is, for the README and `--help`;
+        # a suite's config never says it, the provider owns it.
+        #
+        # @return [String]
+        def request_type_description = ""
+
+        # A direct provider serves a vendor's own API and is reached only
+        # through a `direct:` entry that names it; it never claims an id from
+        # `vendors:`, however the id is spelled.
+        #
+        # @return [Boolean]
+        def direct? = false
+
+        # @return [String] short name, for messages and the leaderboard
+        def name = self.class.name.split("::").last.gsub(/([a-z\d])([A-Z])/, '\1-\2').downcase
+
+        # @param model_id [String] as the lineup names it
+        # @return [Boolean] whether this provider serves the model
+        def serves?(model_id) = model_id.start_with?(prefix)
+
+        # @return [String] the id prefix this provider claims; "" claims everything
+        def prefix = ""
+
+        # @param model_id [String]
+        # @return [String] the id as the provider's API knows it
+        def model_name(model_id) = model_id.delete_prefix(prefix)
+
+        # @return [Array<String>] environment variables a run through this provider needs
+        def env_keys = []
+
+        # @return [Array<String>] the efforts the `-e` flag accepts for this provider's models
+        def efforts = EFFORTS
+
+        # Lists the model ids the provider serves, from the network, so a
+        # misspelled id aborts the run before any experiment starts instead
+        # of failing every case and showing up as a row of errors.
+        #
+        # @return [Array<String>, nil] ids as the lineup names them; nil when the provider cannot list them
+        def catalog = nil
+
+        # Prepares the client once per process, after the env files loaded.
+        #
+        # @option options [String] :app_name what the client tells the API the caller is
+        # @return [void]
+        def setup!(**) = nil
+
+        # @param model [Model]
+        # @param system [String, nil]
+        # @param user [String]
+        # @param json [Boolean]
+        # @return [Completion]
+        def complete(model, system:, user:, json: true)
+          raise NotImplementedError, "#{self.class}#complete must return an Agent::Evals::Completion"
+        end
+
+        # @param contender [Contender]
+        # @return [Model]
+        def model_for(contender) = Model.new(id: contender.model, provider: self, settings: contender.settings)
+
+        # Every id in `model_ids` the provider does not list.
+        #
+        # @param model_ids [Array<String>]
+        # @return [Array<String>]
+        def unknown(model_ids)
+          listed = catalog
+          listed.nil? ? [] : model_ids - listed
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/providers/catalog.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "net/http"
+
+module Agent
+  module Evals
+    module Providers
+      # One GET of a provider's model list, parsed. Every failure becomes an
+      # {Error} that names the URL, so an offline run says so instead of
+      # printing a socket error from inside a provider.
+      module Catalog
+        module_function
+
+        # @param url [String]
+        # @param headers [Hash{String => String}]
+        # @return [Hash] the parsed JSON body
+        # @raise [Error] when the list cannot be read
+        def fetch(url, headers = {})
+          uri = URI(url)
+          response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: 10, read_timeout: 20) do |http|
+            http.request(Net::HTTP::Get.new(uri, headers))
+          end
+          raise Error, "could not list models at #{url}: HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+          JSON.parse(response.body)
+        rescue SocketError, SystemCallError, Net::OpenTimeout, Net::ReadTimeout, JSON::ParserError, KeyError => e
+          raise Error, "could not list models at #{url}: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/providers/open_router.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    module Providers
+      # Every model OpenRouter serves, through RubyLLM, because Braintrust
+      # traces RubyLLM calls (tokens, latency, cost) and not raw HTTP. Claims
+      # every id no other provider does, so it is always asked last.
+      class OpenRouter < Base
+        URL = "https://openrouter.ai/api/v1/models"
+        KEY = "OPENROUTER_API_KEY"
+
+        # OpenRouter's reasoning.effort values.
+        EFFORTS = %w[none minimal low medium high xhigh max].freeze
+
+        def name = "openrouter"
+        def request_type = :prompt
+        def request_type_description = "a chat completion: system and user text in, text out"
+        def env_keys = [KEY]
+        def efforts = EFFORTS
+
+        # @return [Array<String>] every id OpenRouter lists
+        def catalog = Catalog.fetch(URL).fetch("data").map { it.fetch("id") }
+
+        # Points RubyLLM at OpenRouter. Requires it here, not at load, so a
+        # suite that never reaches OpenRouter never needs the gem.
+        #
+        # @param app_name [String]
+        # @return [void]
+        def setup!(app_name:)
+          Agent::Evals.optional!("ruby_llm", needed_for: "calling models through OpenRouter")
+          RubyLLM.configure do |config|
+            config.openrouter_api_key = ENV.fetch(KEY, nil)
+            config.openrouter_app_name = app_name
+          end
+        end
+
+        # @param model [Model]
+        # @param system [String, nil]
+        # @param user [String]
+        # @param json [Boolean] asks for `response_format: json_object`
+        # @return [Completion] text, plus the tokens and the cost OpenRouter billed
+        def complete(model, system:, user:, json: true)
+          Agent::Evals.optional!("ruby_llm", needed_for: "calling models through OpenRouter")
+          chat = RubyLLM.chat(model: model.name, provider: :openrouter, assume_model_exists: true)
+          chat.with_instructions(system) if system
+          chat.with_temperature(model.temperature) unless model.temperature.nil?
+          chat.with_max_output_tokens(model.max_tokens) if model.max_tokens
+          chat.with_thinking(effort: model.effort.to_sym) if model.effort
+          chat.with_provider_options(response_format: { type: "json_object" }) if json
+
+          message = chat.ask(user)
+          Completion.new(
+            text:  message.content.to_s,
+            usage: Usage.new(input_tokens: message.tokens.input.to_i,
+              output_tokens: message.tokens.output.to_i,
+              cost_usd: message.tokens.reported_cost.to_f),
+            raw:   message
+          )
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/providers/registry.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    module Providers
+      # The providers in use. A `vendors:` contender goes to the first
+      # provider that serves its id, asked last-registered first, so a
+      # suite's own transport beats the defaults and OpenRouter, which claims
+      # everything, is always last. A `direct:` contender goes to the
+      # provider it names, and a direct provider is never asked about an id.
+      class Registry
+        include Enumerable
+
+        # @return [Registry] TypeSafe, then OpenRouter for everything else
+        def self.default = new([OpenRouter.new, TypeSafe.new])
+
+        # @param providers [Array<Base>] the last one is asked first
+        def initialize(providers = [])
+          @providers = providers.dup
+        end
+
+        # Registering a provider under a name already taken replaces it, so a
+        # helper and a required file that both register the same provider
+        # leave one, not two that would be asked twice.
+        #
+        # @param provider [Base]
+        # @return [Base] the provider
+        def register(provider)
+          @providers.reject! { it.name == provider.name }
+          @providers << provider
+          provider
+        end
+
+        def each(&) = @providers.reverse.each(&)
+
+        # @param target [Contender, String] a contender, or a `vendors:` model id
+        # @return [Base]
+        # @raise [Error] when no provider serves it
+        def for(target)
+          return direct(target.provider) if target.is_a?(Contender) && target.direct?
+
+          model_id = target.is_a?(Contender) ? target.model : target
+          find { !it.direct? && it.serves?(model_id) } ||
+            raise(Error, "no provider serves #{model_id.inspect}; have #{reject(&:direct?).map(&:name).join(", ")}")
+        end
+
+        # @param name [Symbol, String] a key under `direct:`
+        # @return [Base]
+        # @raise [Error] when no direct provider has the name
+        def direct(name)
+          find { it.direct? && it.name == name.to_s } ||
+            raise(Error, "no direct provider #{name.to_s.inspect}; have #{direct_names.join(", ")}")
+        end
+
+        # @return [Array<String>] the names `direct:` may use
+        def direct_names = select(&:direct?).map(&:name)
+
+        # @param contenders [Array<Contender>]
+        # @return [Array<Base>] each provider the contenders use, once
+        def used_by(contenders) = contenders.map { self.for(it) }.uniq
+
+        # @param contenders [Array<Contender>]
+        # @return [Array<String>] the API keys the contenders' providers need
+        def keys_for(contenders) = used_by(contenders).flat_map(&:env_keys).uniq
+
+        # @param contender [Contender]
+        # @return [Model]
+        def model_for(contender) = self.for(contender).model_for(contender)
+
+        # One refused contender: who, and why.
+        Refusal = Data.define(:contender, :reason) do
+          def to_s = "#{contender.model} refused: #{reason}"
+        end
+
+        # Splits the contenders into those whose provider produces a request
+        # type the suite declares and those it does not. A refused contender
+        # is named, never run, and never scored 0: a model the suite cannot
+        # even ask is not a model that answered wrongly.
+        #
+        # @param contenders [Array<Contender>]
+        # @param request_types [Array<Symbol>] what the suite's config declares
+        # @return [Array(Array<Contender>, Array<Refusal>)] admitted, refused
+        def admit(contenders, request_types:)
+          admitted, refused = contenders.partition { request_types.include?(self.for(it).request_type) }
+          refusals = refused.map do |contender|
+            needs = self.for(contender).request_type
+            Refusal.new(contender: contender, reason: "needs #{needs}; this suite produces #{request_types.join(", ")}")
+          end
+          [admitted, refusals]
+        end
+
+        # @return [Array<String>] every effort any provider accepts, for the flag's help
+        def efforts = flat_map(&:efforts).uniq
+
+        # Asks each provider in use for its catalog, and names every model it
+        # does not list.
+        #
+        # @param contenders [Array<Contender>]
+        # @return [void]
+        # @raise [Error] naming every unknown model
+        def verify!(contenders)
+          unknown = contenders.uniq(&:key).group_by { self.for(it) }.flat_map { |provider, list| provider.unknown(list.map(&:model)) }
+          return if unknown.empty?
+
+          raise Error,
+            "unknown model#{"s" if unknown.size > 1} #{unknown.join(", ")}: not listed by its provider; " \
+            "check the id in .agent-eval.config.yml or -m"
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/providers/type_safe.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    module Providers
+      # TypeSafe's own API, which serves Jev as typed questions (noul, choice,
+      # score) rather than chat: request type `typesafe`, reached only through
+      # a `direct: typesafe:` entry, whose ids read "typesafe/<name>". Jev on
+      # OpenRouter, `typesafe/jev-router` under `vendors:`, is a chat model
+      # and goes through {OpenRouter}; the contender carries which is meant.
+      # The gem knows this API's model list, key and pricing, enough for
+      # `verify` and the cost column; the request shape waits for the first
+      # suite that declares `request_types: [typesafe]`.
+      class TypeSafe < Base
+        PREFIX = "typesafe/"
+        URL = "https://api.typesafe.ai/v1/models"
+        KEY = "TYPESAFE_API_KEY"
+
+        # TypeSafe bills input tokens only: $0.042 per million for jev-latest
+        # and jev-preview (docs.typesafe.ai/models, read 2026-10-14). Output is free.
+        USD_PER_INPUT_TOKEN = 0.042 / 1_000_000
+
+        def name = "typesafe"
+        def request_type = :typesafe
+        def request_type_description = "typed questions: noul, choice, score"
+        def direct? = true
+        def prefix = PREFIX
+        def env_keys = [KEY]
+
+        # Jev takes no reasoning effort.
+        def efforts = []
+
+        # @return [Array<String>] the models, as "typesafe/<name>"
+        def catalog
+          Catalog.fetch(URL, "Authorization" => "Bearer #{ENV.fetch(KEY, nil)}").fetch("models").map { "#{PREFIX}#{it.fetch("name")}" }
+        end
+
+        # @param input_tokens [Integer]
+        # @return [Usage] with the cost at TypeSafe's input rate
+        def usage(input_tokens:, output_tokens: 0)
+          Usage.new(input_tokens: input_tokens, output_tokens: output_tokens, cost_usd: input_tokens * USD_PER_INPUT_TOKEN)
+        end
+
+        # @raise [Error] always: there is no typesafe consumer yet
+        def complete(model, system:, user:, json: true) # rubocop:disable Lint/UnusedMethodArgument
+          raise Error,
+            "#{model.id}: no typesafe consumer yet; the typed request shape is defined by the first suite " \
+            "that declares request_types: [typesafe]"
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/reply.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # What an {Adapter} returns for one case. `error` is the model's failure
+    # (it did not answer the question: no JSON, a refusal) and scores 0; a
+    # transport failure is not the model's and is raised instead, so it is
+    # counted as an error of the run rather than a wrong answer.
+    Reply = Data.define(:output, :raw, :usage, :error, :metadata) do
+      def initialize(output: nil, raw: nil, usage: Usage.zero, error: nil, metadata: {})
+        super(output: output, raw: raw, usage: Usage.from(usage), error: error, metadata: metadata)
+      end
+
+      # @return [Boolean] whether the model failed to answer
+      def error? = !error.nil?
+    end
+  end
+end

data/lib/agent/evals/run_result.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # One case, run once: what came back and what it scored. `reply` is nil
+    # when the adapter raised, and `error` then holds the message.
+    RunResult = Data.define(:experiment, :case, :reply, :score, :duration, :error) do
+      def initialize(experiment:, case:, reply: nil, score: nil, duration: 0.0, error: nil) = super
+
+      # @return [Boolean] whether the adapter raised rather than answered
+      def failed? = !error.nil?
+    end
+  end
+end

data/lib/agent/evals/runtime.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "dotenv"
+
+module Agent
+  module Evals
+    # One-time process setup: secrets from the suite's env files, a check that
+    # every key and gem the run needs is there, the suite's own hooks, and
+    # the providers' clients.
+    class Runtime
+      attr_reader :env_files, :hooks, :app_name
+
+      # @param env_files [Array<String>] absolute paths; those that exist load
+      # @param hooks [Array<Proc>] the suite's `before_run` blocks
+      # @param app_name [String] what a provider tells its API the caller is
+      def initialize(env_files: [], hooks: [], app_name: "agent-evals")
+        @env_files = env_files
+        @hooks = hooks
+        @app_name = app_name
+      end
+
+      # Loads the env files that exist, without overriding what is already
+      # set, so an exported variable beats the file.
+      #
+      # @return [void]
+      def load_env
+        files = env_files.select { File.exist?(it) }
+        Dotenv.load(*files) unless files.empty?
+      end
+
+      # Fails before anything is called when a key the run needs is neither
+      # exported nor in an env file.
+      #
+      # @param names [Array<String>]
+      # @return [void]
+      # @raise [Error] naming every missing variable
+      def require_env!(names)
+        missing = names.uniq.select { ENV[it].to_s.strip.empty? }
+        return if missing.empty?
+
+        found = env_files.select { File.exist?(it) }
+        where = if found.empty?
+                  env_files.empty? ? "no env files are configured" : "there is no #{env_files.join(" or ")}"
+                else
+                  "checked the environment, then #{found.join(", ")}"
+                end
+        raise Error,
+          "missing #{missing.join(", ")}: export them or add them to an env file (#{where}); " \
+          "an exported blank value wins over the file"
+      end
+
+      # @param providers [Array<Providers::Base>] the providers the run's models use
+      # @param tracker [Class<Tracker>, nil] the remote tracker chosen, whose gems and key are checked too
+      # @return [void]
+      # @raise [Error] when a needed variable or gem is missing
+      def setup!(providers: [], tracker: nil)
+        load_env
+        require_env!(providers.flat_map(&:env_keys))
+        tracker&.requirements!(ENV)
+        hooks.each(&:call) # after load_env: a hook usually re-reads ENV
+        providers.each { it.setup!(app_name: app_name) }
+      end
+
+      # Opens a URL in the desktop browser.
+      #
+      # @param url [String]
+      # @return [Boolean] whether the opener ran
+      def browse(url)
+        opener = RUBY_PLATFORM.include?("darwin") ? "open" : "xdg-open"
+        system(opener, url, out: File::NULL, err: File::NULL) || false
+      end
+    end
+  end
+end

data/lib/agent/evals/schema_validator.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # Checks a Hash of answers against a field map, for scorers and verifiers
+    # whose expected values are typed and closed. Nothing here is specific to
+    # what the fields mean.
+    #
+    # The map is `{ "name" => { type:, enum:, nullable: } }`; `type` is one of
+    # {TYPES}, `enum` lists the allowed values for enum and multi_enum, and a
+    # nullable field accepts nil (the text never answered it).
+    class SchemaValidator
+      TYPES = %i[boolean integer number string enum multi_enum].freeze
+
+      attr_reader :fields
+
+      # @param fields [Hash{String, Symbol => Hash}]
+      # @raise [ArgumentError] on a type outside {TYPES}, since a typo there would pass everything
+      def initialize(fields)
+        @fields = fields.to_h { |name, spec| [name.to_s, normalize(name, spec)] }.freeze
+      end
+
+      # @return [Array<String>] the field names
+      def names = fields.keys
+
+      # @param values [Hash] String or Symbol keys
+      # @return [Array<String>] every problem found; empty when the values fit the map
+      def problems(values)
+        return ["#{values.inspect} is not a Hash"] unless values.is_a?(Hash)
+
+        actual = values.transform_keys(&:to_s)
+        missing = names - actual.keys
+        extra = actual.keys - names
+        problems = []
+        problems << "missing #{missing.join(", ")}" if missing.any?
+        problems << "unknown fields #{extra.join(", ")}" if extra.any?
+        fields.each do |name, spec|
+          next unless actual.key?(name)
+
+          problem = value_problem(spec, actual[name])
+          problems << "#{name}: #{problem}" if problem
+        end
+        problems
+      end
+
+      # @param values [Hash]
+      # @return [Boolean]
+      def valid?(values) = problems(values).empty?
+
+      # @param spec [Hash] one field's normalized spec
+      # @param value [Object]
+      # @return [String, nil] what is wrong with the value, or nil
+      def value_problem(spec, value)
+        return (spec[:nullable] ? nil : "is null") if value.nil?
+
+        case spec[:type]
+        when :boolean then "#{value.inspect} is not a boolean" unless [true, false].include?(value)
+        when :integer then "#{value.inspect} is not an integer" unless value.is_a?(Integer)
+        when :number then "#{value.inspect} is not a number" unless value.is_a?(Numeric)
+        when :string then "#{value.inspect} is not a string" unless value.is_a?(String)
+        when :enum then "#{value.inspect} is not one of #{spec[:enum].join("|")}" unless spec[:enum].include?(value)
+        when :multi_enum then multi_enum_problem(spec, value)
+        end
+      end
+
+      private
+
+      def multi_enum_problem(spec, value)
+        return "#{value.inspect} is not an array" unless value.is_a?(Array)
+
+        stray = value - spec[:enum]
+        "#{stray.join(", ")} not in #{spec[:enum].join("|")}" if stray.any?
+      end
+
+      def normalize(name, spec)
+        spec = spec.transform_keys(&:to_sym)
+        type = spec.fetch(:type).to_sym
+        raise ArgumentError, "#{name}: unknown type #{type.inspect}, expected one of #{TYPES.join(", ")}" unless TYPES.include?(type)
+
+        { type: type, enum: Array(spec[:enum] || spec[:values]).map(&:to_s), nullable: spec.fetch(:nullable, false) }
+      end
+    end
+  end
+end