jevalyn 0.1.0

data/lib/jevalyn/question.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module Jevalyn
+  # A single typed question. Jev has exactly three kinds and Jevalyn validates the
+  # shape of each one when the Decision class is defined, so a malformed rubric is a
+  # boot-time error rather than a 422 in production.
+  class Question
+    # Jev accepts up to 255 options on a Choice.
+    MAX_CHOICE_OPTIONS = 255
+
+    # A Score needs at least two levels and takes up to ten.
+    MIN_SCORE_LEVELS = 2
+    MAX_SCORE_LEVELS = 10
+
+    TYPES = %i[noul choice score].freeze
+
+    attr_reader :name, :instructions, :criteria
+
+    def self.build(name, type:, instructions:, criteria: nil)
+      klass = case type.to_sym
+              when :noul   then Noul
+              when :choice then Choice
+              when :score  then Score
+              else
+                raise InvalidQuestionError,
+                      "Unknown question type #{type.inspect} for :#{name}. " \
+                      "Jev supports #{TYPES.map(&:inspect).join(", ")}."
+              end
+
+      klass.new(name, instructions: instructions, criteria: criteria)
+    end
+
+    def initialize(name, instructions:, criteria: nil)
+      @name = name.to_sym
+      @instructions = instructions
+      @criteria = criteria
+      validate!
+      freeze
+    end
+
+    def type = self.class::TYPE
+
+    # The JSON body for this question, as the API wants it.
+    def to_payload
+      payload = { "type" => type.to_s, "instructions" => instructions }
+      payload["criteria"] = criteria_payload unless criteria_payload.nil?
+      payload
+    end
+
+    # Wraps the raw answer hash the API returned for this question.
+    def build_answer(raw)
+      answer_class.new(self, raw)
+    end
+
+    private
+
+    def criteria_payload = criteria
+
+    def validate!
+      validate_instructions!
+      validate_criteria!
+    end
+
+    # The API accepts a string, object or array here -- anything JSON-shaped that
+    # reads as an instruction. What it does not accept is nothing.
+    def validate_instructions!
+      return if instructions.is_a?(String) && !instructions.strip.empty?
+      return if instructions.is_a?(Hash) && !instructions.empty?
+      return if instructions.is_a?(Array) && !instructions.empty?
+
+      raise InvalidQuestionError,
+            "Question :#{name} needs non-empty :instructions (a String, Hash or Array), " \
+            "got #{instructions.inspect}."
+    end
+
+    def validate_criteria!
+      raise NotImplementedError
+    end
+
+    # Probability that the answer to a yes/no question is yes.
+    class Noul < Question
+      TYPE = :noul
+
+      def answer_class = Answer::Noul
+
+      private
+
+      # Optional. When given it explains what a yes and a no mean.
+      def validate_criteria!
+        return if criteria.nil?
+
+        unless criteria.is_a?(Hash)
+          raise InvalidQuestionError,
+                "Question :#{name} is a :noul, so :criteria must be a Hash with " \
+                ":true and/or :false keys, got #{criteria.class}."
+        end
+
+        unknown = criteria.keys.map(&:to_s) - %w[true false]
+        return if unknown.empty?
+
+        raise InvalidQuestionError,
+              "Question :#{name} is a :noul, so :criteria accepts only :true and :false. " \
+              "Unknown key(s): #{unknown.join(", ")}."
+      end
+
+      def criteria_payload
+        return nil if criteria.nil?
+
+        criteria.each_with_object({}) { |(key, value), out| out[key.to_s] = value }
+      end
+    end
+
+    # One option out of a labelled set.
+    class Choice < Question
+      TYPE = :choice
+
+      def answer_class = Answer::Choice
+
+      # The option keys, in declaration order.
+      def options = criteria.keys.map(&:to_sym)
+
+      private
+
+      def validate_criteria!
+        unless criteria.is_a?(Hash) && !criteria.empty?
+          raise InvalidQuestionError,
+                "Question :#{name} is a :choice, so :criteria must be a non-empty Hash of " \
+                "option => description, got #{criteria.inspect}."
+        end
+
+        if criteria.size > MAX_CHOICE_OPTIONS
+          raise InvalidQuestionError,
+                "Question :#{name} declares #{criteria.size} options; Jev accepts at most " \
+                "#{MAX_CHOICE_OPTIONS}."
+        end
+
+        bad = criteria.reject { |_, value| value.nil? || value.is_a?(String) }
+        return if bad.empty?
+
+        raise InvalidQuestionError,
+              "Question :#{name} has non-String descriptions for option(s) " \
+              "#{bad.keys.join(", ")}. Use a String, or nil when the option needs no detail."
+      end
+
+      def criteria_payload
+        criteria.each_with_object({}) { |(key, value), out| out[key.to_s] = value }
+      end
+    end
+
+    # A rating against ordered levels.
+    class Score < Question
+      TYPE = :score
+
+      def answer_class = Answer::Score
+
+      # The level descriptions, lowest first.
+      def levels = criteria.map(&:to_s)
+
+      private
+
+      def validate_criteria!
+        unless criteria.is_a?(Array)
+          raise InvalidQuestionError,
+                "Question :#{name} is a :score, so :criteria must be an ordered Array of " \
+                "level descriptions, got #{criteria.class}."
+        end
+
+        unless criteria.size.between?(MIN_SCORE_LEVELS, MAX_SCORE_LEVELS)
+          raise InvalidQuestionError,
+                "Question :#{name} declares #{criteria.size} level(s); a Jev score takes " \
+                "between #{MIN_SCORE_LEVELS} and #{MAX_SCORE_LEVELS}."
+        end
+
+        bad = criteria.reject { |level| level.is_a?(String) || level.is_a?(Symbol) }
+        return if bad.empty?
+
+        raise InvalidQuestionError,
+              "Question :#{name} has non-String level(s): #{bad.map(&:inspect).join(", ")}."
+      end
+
+      def criteria_payload = levels
+    end
+  end
+end

data/lib/jevalyn/railtie.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "rails/railtie"
+
+module Jevalyn
+  # Wires Jevalyn into a Rails app: the generators, a sensible logger, and
+  # app/decisions as an autoload path so decision classes live next to models.
+  class Railtie < ::Rails::Railtie
+    config.jevalyn = ActiveSupport::OrderedOptions.new if defined?(ActiveSupport::OrderedOptions)
+
+    generators do
+      require "generators/jevalyn/install/install_generator"
+      require "generators/jevalyn/decision/decision_generator"
+      require "generators/jevalyn/guardrail/guardrail_generator"
+    end
+
+    initializer "jevalyn.logger" do
+      Jevalyn.config.logger ||= Rails.logger
+    end
+
+    # A Rails app almost never wants a real API call in its test suite, and a key is
+    # usually absent there anyway. Opt back in per example with `:jevalyn_live`.
+    initializer "jevalyn.mock_mode" do
+      Jevalyn.config.mock_mode = true if Rails.env.test? && Jevalyn.config.api_key.nil?
+    end
+
+    initializer "jevalyn.autoload_paths" do |app|
+      decisions = app.root.join("app/decisions")
+      app.config.autoload_paths << decisions.to_s if decisions.exist?
+    end
+
+    # `rails jevalyn:ping` -- checks the key and the network without leaving the shell.
+    rake_tasks do
+      load File.expand_path("tasks/jevalyn.rake", __dir__)
+    end
+  end
+end

data/lib/jevalyn/result.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module Jevalyn
+  # Everything one evaluation returned: the typed answers, which model answered, and
+  # what it cost. A Result is read-only and safe to pass around or serialise into a job.
+  class Result
+    # What a client sends in the `usage` position when nothing came back.
+    EMPTY_USAGE = { "input_tokens" => 0, "output_tokens" => 0 }.freeze
+
+    attr_reader :answers, :raw, :questions
+
+    # questions  -- Hash of name => Question, in declaration order.
+    # raw        -- the parsed response body, untouched.
+    # thresholds -- Hash of name => confidence floor, already resolved by the Decision.
+    #               A nil floor means that question was not asked to clear anything.
+    def initialize(questions:, raw:, thresholds: nil)
+      @questions = questions
+      @raw = raw || {}
+      @thresholds = normalize_thresholds(thresholds)
+      @answers = build_answers
+    end
+
+    # The versioned model that actually answered, e.g. "jev-1.13.0". Worth logging:
+    # an alias like jev-latest moves under you, and this is what tells you when it did.
+    def model = raw["model"]
+
+    def usage = raw["usage"] || EMPTY_USAGE
+
+    def input_tokens = usage["input_tokens"].to_i
+
+    # Free, as of jev-1.13 -- TypeSafe bills on input tokens only.
+    def output_tokens = usage["output_tokens"].to_i
+
+    def [](name)
+      answer(name).value
+    end
+
+    # The Answer object for a question, rather than its value.
+    def answer(name)
+      @answers.fetch(name.to_sym) do
+        raise UnknownQuestionError,
+              "No answer named #{name.inspect}. This result has: " \
+              "#{@answers.keys.map(&:inspect).join(", ")}."
+      end
+    end
+
+    def key?(name) = @answers.key?(name.to_sym)
+
+    # Plain Hash of question name => unwrapped value.
+    def values
+      @answers.transform_values(&:value)
+    end
+    alias to_h values
+
+    # The floor each question is judged against. Read it to see what a Decision
+    # actually resolved, which is worth logging alongside the answers.
+    attr_reader :thresholds
+
+    def threshold_for(name)
+      @thresholds[name.to_sym]
+    end
+
+    # True when every answer clears its own floor. Pass a number to judge them all
+    # against that one instead. Nouls are measured on how far they sit from a coin
+    # flip, since Jev returns no confidence for them.
+    def certain?(threshold = nil)
+      uncertain_questions(threshold).empty?
+    end
+
+    def uncertain?(threshold = nil) = !certain?(threshold)
+
+    # True when one named answer clears its floor.
+    def certain_for?(name, threshold = :__declared__)
+      threshold = threshold_for(name) if threshold == :__declared__
+
+      answer(name).certain?(threshold)
+    end
+
+    # Names of the answers that fell below their floor -- the ones worth routing to
+    # a human or a slower model.
+    def uncertain_questions(threshold = nil)
+      @answers.reject do |name, answer|
+        answer.certain?(threshold || @thresholds[name])
+      end.keys
+    end
+
+    # The least certain answer's certainty, across every question.
+    def min_certainty
+      @answers.values.filter_map(&:certainty).min
+    end
+
+    # How far each answer sits above (or below) its own floor. Negative means it
+    # missed. Useful for logging which decisions are running close to the line.
+    def certainty_margins
+      @answers.each_with_object({}) do |(name, answer), out|
+        floor = @thresholds[name]
+        certainty = answer.certainty
+        out[name] = floor.nil? || certainty.nil? ? nil : (certainty - floor).round(10)
+      end
+    end
+
+    def each(&) = @answers.each(&)
+
+    include Enumerable
+
+    def inspect
+      pairs = values.map { |name, value| "#{name}=#{value.inspect}" }.join(" ")
+      "#<#{self.class.name.nil? ? "Jevalyn::Result" : self.class.name} #{pairs} model=#{model.inspect}>"
+    end
+
+    # Builds a Result subclass with a reader per question, so a Decision's answers
+    # read as `result.department` rather than `result[:department]`.
+    def self.class_for(questions)
+      Class.new(self) do
+        questions.each_value do |question|
+          name = question.name
+
+          define_method(name) { self[name] }
+          define_method(:"#{name}_answer") { answer(name) }
+          define_method(:"#{name}_certainty") { answer(name).certainty }
+          define_method(:"#{name}_probabilities") { answer(name).probabilities }
+          define_method(:"#{name}_threshold") { threshold_for(name) }
+          define_method(:"#{name}_certain?") { |threshold = :__declared__| certain_for?(name, threshold) }
+          define_method(:"#{name}_uncertain?") { |threshold = :__declared__| !certain_for?(name, threshold) }
+
+          case question.type
+          when :noul
+            define_method(:"#{name}?") { |threshold = 0.5| answer(name).true?(threshold) }
+          when :choice
+            define_method(:"#{name}_confidence") { answer(name).confidence }
+          when :score
+            define_method(:"#{name}_confidence") { answer(name).confidence }
+            define_method(:"#{name}_label") { answer(name).label }
+            define_method(:"#{name}_level") { answer(name).level }
+          end
+        end
+      end
+    end
+
+    private
+
+    # Accepts a Hash of floors, a single number meaning "all of them", or nothing.
+    def normalize_thresholds(thresholds)
+      case thresholds
+      when nil     then questions.keys.to_h { |name| [name, nil] }.freeze
+      when Numeric then questions.keys.to_h { |name| [name, thresholds] }.freeze
+      when Hash
+        questions.keys.to_h { |name| [name, thresholds[name] || thresholds[name.to_s]] }.freeze
+      else
+        raise ConfigurationError,
+              "thresholds must be a Hash of question => floor, a single number, or nil; " \
+              "got #{thresholds.class}."
+      end
+    end
+
+    def build_answers
+      raw_answers = raw["answers"] || {}
+
+      questions.each_with_object({}) do |(name, question), out|
+        out[name] = question.build_answer(raw_answers[name.to_s])
+      end.freeze
+    end
+  end
+end

data/lib/jevalyn/router.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module Jevalyn
+  # Sends work to a handler based on a decision, with a confidence floor underneath.
+  #
+  # Two shapes, and they compose. A plain dispatch table:
+  #
+  #   router = Jevalyn::Router.new do |r|
+  #     r.route :simple_lookup, to: OrderLookup
+  #     r.route :open_ended,    to: llm_client   # anything responding to #call
+  #   end
+  #
+  #   router.dispatch(:simple_lookup, state: order)
+  #
+  # Or a decision-driven router, which is the reason this class exists -- Jev picks
+  # the branch, and anything it is not sure about goes somewhere safer:
+  #
+  #   router = Jevalyn::Router.new(SupportTriage, on: :department) do |r|
+  #     r.route :billing,   to: BillingInbox
+  #     r.route :technical, to: ->(state, result) { Oncall.page(state, result) }
+  #     r.route :sales,     to: SalesInbox
+  #     r.uncertain_below to: HumanQueue
+  #   end
+  #
+  #   router.call(ticket)
+  #
+  # Handlers are called with (state, result) when they take two arguments and (state)
+  # when they take one, so a plain Proc, a Decision, a job class and a service object
+  # all work without adapters.
+  class Router
+    Route = Struct.new(:key, :handler, keyword_init: true)
+
+    attr_reader :decision, :question, :routes
+
+    # decision -- optional Jevalyn::Decision subclass that picks the branch.
+    # on       -- the choice question whose answer names the route. Defaults to the
+    #             decision's only question when it declares just one.
+    def initialize(decision = nil, on: nil)
+      @decision = decision
+      @question = on
+      @routes = {}
+      @fallback = nil
+      @floor = nil
+      @floor_handler = nil
+
+      yield self if block_given?
+
+      validate_question! if decision
+    end
+
+    # Registers a handler under a key.
+    def route(key, to:)
+      @routes[key.to_sym] = Route.new(key: key.to_sym, handler: to)
+      self
+    end
+
+    # Where anything unrouted goes. Without one, an unrouted key raises.
+    def fallback(to:)
+      @fallback = to
+      self
+    end
+
+    # Where an answer that misses its confidence floor goes, whatever the answer was.
+    # This is the confidence-gated half: a wrong-but-confident answer is a routing bug,
+    # a not-confident answer is a known unknown and belongs with a human or a slower
+    # model.
+    #
+    # With no threshold it uses the floor the question itself declares, so the number
+    # lives in one place:
+    #
+    #   r.uncertain_below to: HumanQueue          # the decision's own floor
+    #   r.uncertain_below 0.9, to: HumanQueue     # stricter, just for this router
+    def uncertain_below(threshold = nil, to:)
+      @floor = threshold
+      @floor_handler = to
+      self
+    end
+
+    # Calls a registered handler directly, skipping the decision.
+    def dispatch(key, state:, result: nil)
+      registered = @routes[key.to_sym]
+
+      unless registered
+        return invoke(@fallback, state, result) if @fallback
+
+        raise UnknownQuestionError,
+              "No route for #{key.inspect}. Registered: #{@routes.keys.map(&:inspect).join(", ")}."
+      end
+
+      invoke(registered.handler, state, result)
+    end
+
+    # Runs the decision, then dispatches on its answer.
+    def call(state, **options)
+      unless decision
+        raise ConfigurationError,
+              "This Router has no decision, so it can only #dispatch(key, state:). " \
+              "Build it as Router.new(SomeDecision, on: :question) to use #call."
+      end
+
+      result = decision.evaluate(state, **options)
+      answer = result.answer(question_name)
+
+      floor = @floor || result.threshold_for(question_name)
+      return invoke(@floor_handler, state, result) if @floor_handler && answer.uncertain?(floor)
+
+      dispatch(answer.value, state: state, result: result)
+    end
+
+    # The decision's answer without dispatching -- useful in specs and consoles.
+    def decide(state, **options)
+      decision.evaluate(state, **options)
+    end
+
+    def keys = @routes.keys
+
+    private
+
+    def question_name
+      @question || decision.question_names.first
+    end
+
+    def validate_question!
+      name = question_name
+
+      unless decision.questions.key?(name)
+        raise ConfigurationError,
+              "#{decision.name} declares no question named #{name.inspect}. " \
+              "It has: #{decision.question_names.map(&:inspect).join(", ")}."
+      end
+
+      declared = decision.questions[name]
+
+      unless declared.type == :choice
+        raise ConfigurationError,
+              "Router routes on a :choice question; #{decision.name}##{name} is a " \
+              ":#{declared.type}."
+      end
+
+      unrouted = declared.options - @routes.keys
+      return if unrouted.empty? || @fallback || @floor_handler
+
+      raise ConfigurationError,
+            "#{decision.name}##{name} can answer #{unrouted.map(&:inspect).join(", ")}, " \
+            "which this Router has no route for. Add them, or a `fallback to:`."
+    end
+
+    # Handlers vary in shape on purpose: a Decision subclass, a Proc that only wants
+    # the state, and a service object that wants both should all work unadapted.
+    def invoke(handler, state, result)
+      return handler.evaluate(state) if handler.is_a?(Class) && handler <= Decision
+
+      unless handler.respond_to?(:call)
+        raise ConfigurationError,
+              "Route handler #{handler.inspect} must respond to #call, or be a " \
+              "Jevalyn::Decision subclass."
+      end
+
+      return handler.call(state, result) if accepts_two?(handler)
+
+      handler.call(state)
+    end
+
+    def accepts_two?(handler)
+      method = handler.respond_to?(:parameters) ? handler : handler.method(:call)
+      parameters = method.parameters
+
+      return true if parameters.any? { |type, _| type == :rest }
+
+      parameters.count { |type, _| %i[req opt].include?(type) } >= 2
+    rescue NameError
+      false
+    end
+  end
+end

data/lib/jevalyn/state.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Jevalyn
+  # Turns whatever you hand a Decision into something the API accepts as `state`:
+  # a String, an Object or an Array.
+  #
+  # The convention is one method. If an object responds to #jevalyn_state, that is the
+  # state -- which is how a model says "send these five columns, not all forty".
+  # Otherwise Jevalyn falls back to #as_json, then to #to_s.
+  module State
+    # Jev's per-request budget: 64k tokens for state plus every question, and 32k for
+    # state plus the single longest question. Tokens are not characters, so this is a
+    # deliberately loose guard against posting a whole database row set by accident.
+    ROUGH_CHARS_PER_TOKEN = 4
+    SOFT_CHAR_LIMIT = 32_000 * ROUGH_CHARS_PER_TOKEN
+
+    module_function
+
+    def serialize(object)
+      state = coerce(object)
+
+      if state.nil? || (state.respond_to?(:empty?) && state.empty?)
+        raise ConfigurationError,
+              "State is empty. Jev needs something to evaluate -- pass a String, a Hash, " \
+              "an Array, or an object that responds to #jevalyn_state."
+      end
+
+      state
+    end
+
+    def coerce(object)
+      case object
+      when nil            then nil
+      when String         then object
+      when Symbol         then object.to_s
+      # Numbers and booleans are valid JSON and read better to the model as
+      # themselves than as quoted strings.
+      when Numeric, true, false then object
+      when Hash           then stringify(object)
+      when Array          then object.map { |item| coerce(item) }
+      else
+        coerce_object(object)
+      end
+    end
+
+    # Rough token estimate, for logging and for the oversize warning. Not exact --
+    # only the API knows the real count, and it reports it back in `usage`.
+    def estimated_tokens(state)
+      json = state.is_a?(String) ? state : JSON.generate(state)
+      (json.length / ROUGH_CHARS_PER_TOKEN.to_f).ceil
+    end
+
+    def oversized?(state)
+      json = state.is_a?(String) ? state : JSON.generate(state)
+      json.length > SOFT_CHAR_LIMIT
+    end
+
+    def coerce_object(object)
+      return coerce(object.jevalyn_state) if object.respond_to?(:jevalyn_state)
+
+      if defined?(::ActiveRecord::Base) && object.is_a?(::ActiveRecord::Base)
+        return StateAdapters::ActiveRecordAdapter.serialize(object)
+      end
+
+      if defined?(::ActiveRecord::Relation) && object.is_a?(::ActiveRecord::Relation)
+        return object.map { |record| coerce(record) }
+      end
+
+      return coerce(object.as_json) if object.respond_to?(:as_json)
+
+      object.to_s
+    end
+
+    def stringify(hash)
+      hash.each_with_object({}) { |(key, value), out| out[key.to_s] = coerce(value) }
+    end
+  end
+end

data/lib/jevalyn/state_adapters/active_record_adapter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Jevalyn
+  module StateAdapters
+    # Serialises an ActiveRecord model into the `state` field.
+    #
+    # The default is `record.as_json`, which sends every column. That is usually the
+    # wrong thing: it burns tokens on ids and timestamps Jev has no use for, and it
+    # ships PII to a third party that did not need it. Narrow it with `only:`, or
+    # define #jevalyn_state on the model and forget the adapter exists.
+    class ActiveRecordAdapter
+      # Columns that are noise in almost every decision.
+      DEFAULT_EXCEPT = %w[id created_at updated_at].freeze
+
+      def self.serialize(record, only: nil, except: nil)
+        return record.jevalyn_state if only.nil? && except.nil? && record.respond_to?(:jevalyn_state)
+
+        options = {}
+        options[:only] = Array(only).map(&:to_s) if only
+        options[:except] = Array(except).map(&:to_s) if except
+
+        State.stringify(record.as_json(**options))
+      end
+    end
+  end
+end