jevalyn 0.1.0

data/lib/jevalyn/decision.rb

@@ -0,0 +1,238 @@
+# frozen_string_literal: true
+
+module Jevalyn
+  # A named set of questions your app asks about a piece of state.
+  #
+  #   class SupportTriage < Jevalyn::Decision
+  #     question :urgent, type: :noul,
+  #       instructions: "Does this convey urgency?"
+  #
+  #     question :department, type: :choice,
+  #       instructions: "Which team should handle this?",
+  #       criteria: {
+  #         billing:   "Payments, invoicing, refunds",
+  #         technical: "Bugs, outages, integrations",
+  #         sales:     "Pricing, upgrades, new accounts"
+  #       }
+  #
+  #     confidence_threshold 0.75
+  #   end
+  #
+  #   result = SupportTriage.evaluate(ticket.body)
+  #   result.department   # => :technical
+  #   result.certain?     # => true
+  #
+  # Every question is validated when the class body runs, so a rubric with eleven
+  # score levels fails on boot rather than as a 422 on a Friday afternoon.
+  class Decision
+    class << self
+      # Declares one question. Keys map one-to-one onto the API's question object,
+      # except `confidence_threshold`, which is Jevalyn's and is never sent.
+      def question(name, type:, instructions:, criteria: nil, confidence_threshold: nil)
+        name = name.to_sym
+
+        if questions.key?(name)
+          raise InvalidQuestionError,
+                "#{self.name || "Decision"} already declares a question named :#{name}."
+        end
+
+        own_questions[name] = Question.build(name, type: type, instructions: instructions, criteria: criteria)
+        unless confidence_threshold.nil?
+          own_thresholds[name] =
+            validate_threshold!(confidence_threshold,
+                                "confidence_threshold for :#{name}")
+        end
+        reset_result_class!
+        own_questions[name]
+      end
+
+      # Sugar for the three types. `noul :urgent, "Does this convey urgency?"`
+      def noul(name, instructions, criteria: nil, confidence_threshold: nil)
+        question(name, type: :noul, instructions: instructions, criteria: criteria,
+                       confidence_threshold: confidence_threshold)
+      end
+
+      def choice(name, instructions, criteria, confidence_threshold: nil)
+        question(name, type: :choice, instructions: instructions, criteria: criteria,
+                       confidence_threshold: confidence_threshold)
+      end
+
+      def score(name, instructions, criteria, confidence_threshold: nil)
+        question(name, type: :score, instructions: instructions, criteria: criteria,
+                       confidence_threshold: confidence_threshold)
+      end
+
+      # Reads or sets the default confidence floor for this decision's questions.
+      # A question that declares its own overrides this; see #confidence_threshold_for.
+      # Called with no argument it reads; the inherited or global value is the fallback.
+      def confidence_threshold(value = :__read__)
+        if value == :__read__
+          return @confidence_threshold if defined?(@confidence_threshold) && @confidence_threshold
+          return superclass.confidence_threshold if superclass.respond_to?(:confidence_threshold)
+
+          return Jevalyn.config.default_confidence_threshold
+        end
+
+        @confidence_threshold = validate_threshold!(value, "confidence_threshold")
+      end
+
+      # The floor one question is judged against: its own if it declared one, this
+      # decision's default otherwise, and the global default under that.
+      def confidence_threshold_for(name)
+        declared = thresholds_by_question[name.to_sym]
+        return declared unless declared.nil?
+
+        confidence_threshold
+      end
+
+      # Every question's resolved floor, which is what a Result is judged against.
+      def thresholds
+        questions.keys.to_h { |name| [name, confidence_threshold_for(name)] }
+      end
+
+      # Per-question floors declared on this class and its ancestors.
+      def thresholds_by_question
+        inherited = superclass.respond_to?(:thresholds_by_question) ? superclass.thresholds_by_question : {}
+        inherited.merge(own_thresholds)
+      end
+
+      # Pins this decision to a specific model. Worth doing once you have tuned
+      # thresholds against a version -- `jev-latest` moves without telling you.
+      def model(value = :__read__)
+        if value == :__read__
+          return @model if defined?(@model) && @model
+          return superclass.model if superclass.respond_to?(:model)
+
+          return nil
+        end
+
+        @model = value
+      end
+
+      # All questions, inherited ones first.
+      def questions
+        inherited = superclass.respond_to?(:questions) ? superclass.questions : {}
+        inherited.merge(own_questions)
+      end
+
+      def question_names = questions.keys
+
+      # Evaluates the state and returns a Jevalyn::Result with a reader per question.
+      #
+      # Two ways to override the declared floors for one call. `confidence_threshold:`
+      # applies one number to every question; `thresholds:` names them individually and
+      # wins over the blanket value where both are given.
+      #
+      #   SupportTriage.evaluate(body, confidence_threshold: 0.95)
+      #   SupportTriage.evaluate(body, thresholds: { department: 0.9 })
+      def evaluate(state, model: nil, confidence_threshold: :__default__, thresholds: nil,
+                   client: Jevalyn.client)
+        ensure_questions!
+
+        client.evaluate(
+          state: state,
+          questions: questions,
+          model: model || self.model,
+          thresholds: resolve_thresholds(confidence_threshold, thresholds),
+          result_class: result_class,
+          decision: self
+        )
+      end
+
+      # Same call, on an ActiveJob queue. The block runs with the Result once the job
+      # completes; see Jevalyn::EvaluationJob for the handler contract.
+      def evaluate_later(state, on:, model: nil, queue: nil, **job_options)
+        ensure_questions!
+        EvaluationJob.enqueue(
+          decision: self, state: state, handler: on, model: model, queue: queue, **job_options
+        )
+      end
+
+      # The request body that `evaluate` would send, without sending it. Handy in a
+      # console for checking token cost before wiring a decision into a hot path.
+      def payload_for(state, model: nil)
+        {
+          "model" => model || self.model || Jevalyn.config.default_model,
+          "state" => State.serialize(state),
+          "questions" => questions.each_with_object({}) { |(name, q), out| out[name.to_s] = q.to_payload }
+        }
+      end
+
+      # Rough input-token estimate for a given state. The real number comes back in
+      # Result#input_tokens; this is for sizing things up beforehand.
+      def estimated_tokens(state)
+        State.estimated_tokens(payload_for(state))
+      end
+
+      def result_class
+        @result_class ||= build_result_class
+      end
+
+      # Overridden by Guardrail to bolt #allow? onto the result.
+      def build_result_class
+        Result.class_for(questions)
+      end
+
+      def own_questions
+        @own_questions ||= {}
+      end
+
+      def own_thresholds
+        @own_thresholds ||= {}
+      end
+
+      private
+
+      def inherited(subclass)
+        super
+        subclass.instance_variable_set(:@own_questions, {})
+        subclass.instance_variable_set(:@own_thresholds, {})
+      end
+
+      # Call-site overrides, least specific first: the declared floors, then a blanket
+      # value applied to every question, then per-question values on top of that.
+      def resolve_thresholds(blanket, per_question)
+        resolved =
+          if blanket == :__default__
+            thresholds
+          else
+            validate_threshold!(blanket, "confidence_threshold")
+            questions.keys.to_h { |name| [name, blanket] }
+          end
+
+        return resolved if per_question.nil?
+
+        per_question.each_with_object(resolved.dup) do |(name, value), out|
+          name = name.to_sym
+
+          unless questions.key?(name)
+            raise ConfigurationError,
+                  "#{self.name || "This Decision"} declares no question named #{name.inspect}. " \
+                  "It has: #{question_names.map(&:inspect).join(", ")}."
+          end
+
+          out[name] = validate_threshold!(value, name.inspect)
+        end
+      end
+
+      def validate_threshold!(value, label)
+        return value if value.nil? || (value.is_a?(Numeric) && value.between?(0, 1))
+
+        raise ConfigurationError,
+              "#{label} must be nil or a number between 0 and 1, got #{value.inspect}."
+      end
+
+      def ensure_questions!
+        return unless questions.empty?
+
+        raise ConfigurationError,
+              "#{name || "This Decision"} declares no questions. Add at least one with " \
+              "`question :name, type: :noul, instructions: \"...\"`."
+      end
+
+      def reset_result_class!
+        @result_class = nil
+      end
+    end
+  end
+end

data/lib/jevalyn/errors.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Jevalyn
+  # Base class for everything Jevalyn raises.
+  class Error < StandardError; end
+
+  # Raised when the gem is asked to do something it has not been configured for
+  # (missing API key, unknown question type, a Decision that declares no questions).
+  class ConfigurationError < Error; end
+
+  # Raised at class-definition time when a Decision declares a malformed question.
+  class InvalidQuestionError < ConfigurationError; end
+
+  # Raised when a Result is asked for a question that was never declared.
+  class UnknownQuestionError < Error; end
+
+  # The request never completed.
+  class TimeoutError < Error; end
+
+  # The request could not reach the API at all (DNS, TLS, refused connection).
+  class ConnectionError < Error; end
+
+  # The API answered with a non-2xx status.
+  class APIError < Error
+    attr_reader :status, :body, :response_headers
+
+    def initialize(message = nil, status: nil, body: nil, response_headers: nil)
+      @status = status
+      @body = body
+      @response_headers = response_headers || {}
+      super(message || default_message)
+    end
+
+    # Builds the most specific error class for a given HTTP status.
+    def self.from_response(status:, body:, headers: {})
+      klass = case status
+              when 401 then AuthenticationError
+              when 403 then PermissionDeniedError
+              when 404 then NotFoundError
+              when 422 then InvalidRequestError
+              when 429 then RateLimitError
+              when 529 then OverloadedError
+              when 500..599 then ServerError
+              else self
+              end
+
+      klass.new(extract_message(body), status: status, body: body, response_headers: headers)
+    end
+
+    def self.extract_message(body)
+      return body if body.is_a?(String)
+      return nil unless body.is_a?(Hash)
+
+      error = body["error"]
+      return error if error.is_a?(String)
+      return error["message"] if error.is_a?(Hash) && error["message"]
+
+      body["message"] || body["detail"]
+    end
+
+    # True when retrying the identical request has a reasonable chance of succeeding.
+    def retryable?
+      false
+    end
+
+    private
+
+    def default_message = "TypeSafe API returned HTTP #{status}"
+  end
+
+  # 401 -- missing or invalid API key.
+  class AuthenticationError < APIError
+    private
+
+    def default_message
+      "TypeSafe rejected the API key. Check Jevalyn.config.api_key / ENV[\"TYPESAFE_API_KEY\"]."
+    end
+  end
+
+  # 403 -- the key is valid but not allowed to do this.
+  class PermissionDeniedError < APIError; end
+
+  # 404 -- unknown endpoint or model.
+  class NotFoundError < APIError; end
+
+  # 422 -- the request body failed validation. The body names the offending field.
+  class InvalidRequestError < APIError
+    private
+
+    def default_message = "TypeSafe rejected the request body (HTTP 422)"
+  end
+
+  # 429 -- over the account's tokens-per-second or requests-per-minute limit.
+  class RateLimitError < APIError
+    def retryable? = true
+
+    # Seconds the API asked us to wait, when it said.
+    def retry_after
+      value = response_headers["retry-after"] || response_headers["Retry-After"]
+      Float(value)
+    rescue ArgumentError, TypeError
+      nil
+    end
+
+    private
+
+    def default_message = "TypeSafe rate limit exceeded (HTTP 429)"
+  end
+
+  # 529 -- TypeSafe is temporarily overloaded.
+  class OverloadedError < APIError
+    def retryable? = true
+
+    private
+
+    def default_message = "TypeSafe is overloaded (HTTP 529)"
+  end
+
+  # 5xx other than 529.
+  class ServerError < APIError
+    def retryable? = true
+  end
+end

data/lib/jevalyn/evaluation_job.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Jevalyn
+  # Runs a Decision on an ActiveJob queue.
+  #
+  # Jev answers in well under a second, so the default advice is to call `evaluate`
+  # inline and keep the decision in the request's control flow -- that is the whole
+  # point of a System One model. Reach for `evaluate_later` when the decision is not
+  # on the critical path: backfilling triage over old tickets, scoring a batch
+  # overnight, or anywhere a third-party outage must not take a request down with it.
+  #
+  #   SupportTriage.evaluate_later(ticket, on: TicketRouter)
+  #   SupportTriage.evaluate_later(ticket, on: "TicketRouter.route")
+  #
+  # The handler is called with (result, state). It is named rather than passed as a
+  # block because a block cannot be serialised onto a queue.
+  module EvaluationJob
+    class << self
+      def enqueue(decision:, state:, handler:, model: nil, queue: nil, **options)
+        ensure_active_job!
+
+        job_class.set(queue: queue || Jevalyn.config.job_queue_name, **options).perform_later(
+          decision.name,
+          serializable(state),
+          handler_name(handler),
+          model
+        )
+      end
+
+      def job_class
+        @job_class ||= build_job_class
+      end
+
+      private
+
+      def ensure_active_job!
+        return if defined?(::ActiveJob::Base)
+
+        raise ConfigurationError,
+              "evaluate_later needs ActiveJob, which is not loaded. Use `evaluate` for " \
+              "an inline call -- Jev answers fast enough to sit in a request."
+      end
+
+      # A Decision's state has to survive a round trip through the queue. GlobalID
+      # handles ActiveRecord; everything else has to already be JSON-shaped.
+      def serializable(state)
+        return state if state.is_a?(String) || state.is_a?(Hash) || state.is_a?(Array)
+        return state if defined?(::GlobalID) && state.respond_to?(:to_global_id)
+
+        State.serialize(state)
+      end
+
+      def handler_name(handler)
+        return handler if handler.is_a?(String)
+        return handler.name if handler.is_a?(Class) || handler.is_a?(Module)
+
+        raise ConfigurationError,
+              "`on:` must be a class, module, or a \"ClassName.method\" string -- a job " \
+              "handler has to survive serialisation, so it cannot be a block or a lambda."
+      end
+
+      def build_job_class
+        klass = Class.new(::ActiveJob::Base) do
+          def perform(decision_name, state, handler_name, model = nil)
+            decision = Object.const_get(decision_name)
+            result = decision.evaluate(state, model: model)
+
+            target, method_name = handler_name.split(".", 2)
+            Object.const_get(target).public_send(method_name || "call", result, state)
+          end
+        end
+
+        # Named so backends that serialise by class name have something stable to use.
+        Jevalyn.const_set(:Job, klass) unless Jevalyn.const_defined?(:Job)
+        klass
+      end
+    end
+  end
+end

data/lib/jevalyn/guardrail.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Jevalyn
+  # A Decision narrowed to one job: should this be allowed through?
+  #
+  #   class ToolCallGuardrail < Jevalyn::Guardrail
+  #     question :safe_to_execute, type: :noul,
+  #       instructions: "Is this tool call safe to run without human review?"
+  #
+  #     allow_above 0.9
+  #   end
+  #
+  #   ToolCallGuardrail.check(tool_call).allow?   # => false
+  #
+  # A guardrail declares exactly one noul question. Jev returns no confidence for a
+  # noul -- the probability *is* the answer -- so the gate is the probability itself,
+  # and `allow_above` is where you set it. The default is 0.5, which is a coin flip
+  # and almost certainly not what you want in front of anything destructive.
+  class Guardrail < Decision
+    DEFAULT_ALLOW_ABOVE = 0.5
+
+    # A guardrail that fails open is worse than one that fails loudly, so an error
+    # from the API denies rather than allows.
+    DEFAULT_ON_ERROR = :deny
+
+    class << self
+      # Probability the noul must reach for #allow? to be true.
+      def allow_above(value = :__read__)
+        if value == :__read__
+          return @allow_above if defined?(@allow_above) && @allow_above
+          return superclass.allow_above if superclass.respond_to?(:allow_above)
+
+          return DEFAULT_ALLOW_ABOVE
+        end
+
+        unless value.is_a?(Numeric) && value.between?(0, 1)
+          raise ConfigurationError,
+                "allow_above must be a number between 0 and 1, got #{value.inspect}."
+        end
+
+        @allow_above = value
+      end
+
+      # What to do when the API call itself fails: :deny (default) or :raise.
+      def on_error(value = :__read__)
+        if value == :__read__
+          return @on_error if defined?(@on_error) && @on_error
+          return superclass.on_error if superclass.respond_to?(:on_error)
+
+          return DEFAULT_ON_ERROR
+        end
+
+        unless %i[deny raise].include?(value)
+          raise ConfigurationError, "on_error must be :deny or :raise, got #{value.inspect}."
+        end
+
+        @on_error = value
+      end
+
+      # Runs the guardrail. Returns a Result answering #allow? and #deny?.
+      def check(state, **options)
+        evaluate(state, **options)
+      rescue APIError, TimeoutError, ConnectionError => e
+        raise if on_error == :raise
+
+        Jevalyn.logger&.warn("[jevalyn] #{name} denied by default: #{e.class}: #{e.message}")
+        denied_result(e)
+      end
+
+      def evaluate(...)
+        ensure_single_noul!
+        super
+      end
+
+      def build_result_class
+        decision = self
+
+        Class.new(super) do
+          # The gate is read at call time so `allow_above` can be changed after the
+          # class body has run -- in an initializer, or per environment.
+          define_method(:threshold) { decision.allow_above }
+
+          # The raw probability the gate is compared against.
+          define_method(:probability) { answer(decision.question_names.first).value }
+
+          def allow?
+            value = probability
+            !value.nil? && value >= threshold
+          end
+
+          def deny? = !allow?
+
+          # Set when the guardrail denied because the call failed, not because Jev said no.
+          attr_reader :error
+
+          def failed? = !error.nil?
+        end
+      end
+
+      private
+
+      # A denial the caller can treat like any other, carrying the cause.
+      def denied_result(error)
+        question_key = question_names.first.to_s
+        raw = { "answers" => { question_key => { "type" => "noul", "noul" => 0.0 } } }
+
+        result = result_class.new(questions: questions, raw: raw, thresholds: thresholds)
+        result.instance_variable_set(:@error, error)
+        result
+      end
+
+      def ensure_single_noul!
+        declared = questions.values
+
+        if declared.size != 1
+          raise ConfigurationError,
+                "#{name || "A Guardrail"} must declare exactly one question, found " \
+                "#{declared.size}. Use a Jevalyn::Decision when you need more than one."
+        end
+
+        return if declared.first.type == :noul
+
+        raise ConfigurationError,
+              "#{name || "A Guardrail"} question :#{declared.first.name} is a " \
+              ":#{declared.first.type}. A guardrail asks one yes/no question, so it " \
+              "must be a :noul."
+      end
+    end
+  end
+end