jevalyn 0.1.0

data/lib/generators/jevalyn/decision/templates/decision.rb.tt

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# <%= class_name %> asks Jev a fixed set of questions about a piece of state and
+# gets typed answers back. It does not generate text -- every answer is a
+# probability, one of your options, or a level on your rubric.
+class <%= class_name %> < Jevalyn::Decision
+<% parsed_questions.each do |name, type| -%>
+<% case type
+   when :noul -%>
+  # A noul returns the probability that the answer is yes, 0.0 to 1.0. You pick the
+  # cutoff -- `result.<%= name %>?` uses 0.5, `result.<%= name %>?(0.9)` is stricter.
+  question :<%= name %>, type: :noul,
+    instructions: "TODO: a yes/no question about the state",
+    criteria: {
+      true: "TODO: what a yes looks like",
+      false: "TODO: what a no looks like"
+    }
+
+<% when :choice -%>
+  # A choice picks one option and returns the full probability distribution plus a
+  # confidence. Give it every real option, not a shortlist -- Jev takes up to 255 and
+  # each one costs only a few tokens.
+  question :<%= name %>, type: :choice,
+    instructions: "TODO: what should the model decide?",
+    criteria: {
+      first_option: "TODO: when this option applies",
+      second_option: "TODO: when this one does instead"
+    },
+    # Optional. Without it this question uses the decision's floor below. Set it when
+    # this answer costs more to get wrong than the others.
+    confidence_threshold: <%= threshold %>
+
+<% when :score -%>
+  # A score rates the state against ordered levels, lowest first, and returns a
+  # weighted number that can land between them: 1.6 means past the second level and
+  # not quite the third. Use `result.<%= name %>_label` for the nearest level's name.
+  question :<%= name %>, type: :score,
+    instructions: "TODO: what should the model rate?",
+    criteria: [
+      "TODO: lowest level",
+      "TODO: middle level",
+      "TODO: highest level"
+    ],
+    confidence_threshold: <%= threshold %>
+
+<% end -%>
+<% end -%>
+  # The default floor for any question above that did not declare its own. Answers
+  # below their floor are reported by `result.uncertain?` and named by
+  # `result.uncertain_questions`. Set these by what a wrong answer costs: a misrouted
+  # ticket is cheap, an auto-approved refund is not.
+  confidence_threshold <%= threshold %>
+end

data/lib/generators/jevalyn/decision/templates/decision_spec.rb.tt

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe <%= class_name %> do
+  it "declares the questions it sends" do
+    expect(described_class.question_names).to contain_exactly(<%= parsed_questions.map { |n, _| ":#{n}" }.join(", ") %>)
+  end
+
+<% first_name, first_type = parsed_questions.first -%>
+  it "answers <%= first_name %>" do
+    stub_jevalyn(described_class, <%= parsed_questions.map { |n, t| "#{n}: #{{ noul: "true", choice: ":first_option", score: "1" }[t]}" }.join(", ") %>)
+
+    result = described_class.evaluate("TODO: a realistic piece of state")
+
+<% case first_type
+   when :noul -%>
+    expect(result.<%= first_name %>?).to be(true)
+<% when :choice -%>
+    expect(result.<%= first_name %>).to eq(:first_option)
+<% when :score -%>
+    expect(result.<%= first_name %>_level).to eq(1)
+<% end -%>
+    expect(described_class).to have_been_evaluated.once
+  end
+
+  # Jev's answers move with the model and with your rubric wording. This is the test
+  # worth keeping honest: a handful of real inputs whose answer you are sure of, run
+  # against the live API. Tag it :jevalyn_live and run it deliberately, not in CI.
+  #
+  # it "routes real examples correctly", :jevalyn_live do
+  #   result = described_class.evaluate(File.read("spec/fixtures/<%= file_name %>.txt"))
+  #   expect(result.<%= first_name %>).to eq(...)
+  # end
+end

data/lib/generators/jevalyn/guardrail/guardrail_generator.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "rails/generators/named_base"
+
+module Jevalyn
+  module Generators
+    # rails g jevalyn:guardrail ToolCall
+    class GuardrailGenerator < ::Rails::Generators::NamedBase
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates a Jevalyn::Guardrail in app/decisions, and a spec for it."
+
+      class_option :question, type: :string, default: "safe",
+                              desc: "Name of the single noul question"
+      class_option :allow_above, type: :numeric, default: 0.9,
+                                 desc: "Probability required to allow"
+      class_option :spec, type: :boolean, default: true
+
+      def create_guardrail
+        template "guardrail.rb.tt", File.join("app/decisions", class_path, "#{file_name}.rb")
+      end
+
+      def create_spec
+        return unless options[:spec]
+
+        template "guardrail_spec.rb.tt", File.join("spec/decisions", class_path, "#{file_name}_spec.rb")
+      end
+
+      private
+
+      def question_name = options[:question]
+
+      def allow_above = options[:allow_above]
+    end
+  end
+end

data/lib/generators/jevalyn/guardrail/templates/guardrail.rb.tt

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# <%= class_name %> is a one-question gate: it asks Jev whether something should be
+# allowed through, and answers #allow? / #deny?.
+#
+#   <%= class_name %>.check(payload).allow?
+class <%= class_name %> < Jevalyn::Guardrail
+  question :<%= question_name %>, type: :noul,
+    instructions: "TODO: is this safe to allow without human review?",
+    criteria: {
+      true: "TODO: what clearly safe looks like",
+      false: "TODO: what needs a human"
+    }
+
+  # Jev returns no confidence for a noul -- the probability is the answer -- so this
+  # is the probability #allow? requires. The default is 0.5, a coin flip; set it by
+  # what letting a bad one through actually costs.
+  allow_above <%= allow_above %>
+
+  # If the API call itself fails, deny rather than allow. Set :raise to handle the
+  # outage yourself instead.
+  on_error :deny
+end

data/lib/generators/jevalyn/guardrail/templates/guardrail_spec.rb.tt

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe <%= class_name %> do
+  it "allows when Jev is confident it is safe" do
+    stub_jevalyn(described_class, <%= question_name %>: 0.99)
+
+    expect(described_class.check("TODO: something safe")).to be_allow
+  end
+
+  it "denies when the probability is below the gate" do
+    stub_jevalyn(described_class, <%= question_name %>: <%= [allow_above.to_f - 0.1, 0.0].max.round(2) %>)
+
+    expect(described_class.check("TODO: something borderline")).to be_deny
+  end
+
+  it "denies rather than failing open when the API is unreachable" do
+    allow(Jevalyn.client).to receive(:evaluate).and_raise(Jevalyn::TimeoutError, "boom")
+
+    result = described_class.check("anything")
+
+    expect(result).to be_deny
+    expect(result).to be_failed
+  end
+end

data/lib/generators/jevalyn/install/install_generator.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "rails/generators/base"
+
+module Jevalyn
+  module Generators
+    # rails g jevalyn:install
+    class InstallGenerator < ::Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates config/initializers/jevalyn.rb and an app/decisions directory."
+
+      class_option :api_key_env, type: :string, default: "TYPESAFE_API_KEY",
+                                 desc: "Environment variable holding the TypeSafe API key"
+
+      def create_initializer
+        template "jevalyn.rb.tt", "config/initializers/jevalyn.rb"
+      end
+
+      def create_decisions_directory
+        empty_directory "app/decisions"
+        create_file "app/decisions/.keep" unless File.exist?(File.join(destination_root,
+                                                                       "app/decisions/.keep"))
+      end
+
+      def report
+        say ""
+        say "Jevalyn is installed.", :green
+        say ""
+        say "  1. Put your key in the environment:  #{options[:api_key_env]}=ts_..."
+        say "  2. Generate a decision:              rails g jevalyn:decision SupportTriage"
+        say "  3. Check the connection:             rails jevalyn:ping"
+        say ""
+      end
+    end
+  end
+end

data/lib/generators/jevalyn/install/templates/jevalyn.rb.tt

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+Jevalyn.configure do |c|
+  # Your TypeSafe API key. Keep it out of the repo.
+  c.api_key = ENV["<%= options[:api_key_env] %>"]
+
+  # "jev-latest" tracks the newest stable release. An alias moves without telling
+  # you, so once you have tuned thresholds against a version, pin it here instead
+  # (e.g. "jev-1.13.0"). Every Result reports the version that actually answered.
+  c.default_model = "jev-latest"
+
+  # Jev answers in well under a second. A long timeout only means a slow request
+  # holds a Rails thread open for longer than it is worth.
+  c.timeout = 10
+  c.open_timeout = 5
+
+  # 429 (rate limited) and 529 (overloaded) are retried with exponential backoff,
+  # honouring the API's retry-after header when it sends one.
+  c.max_retries = 2
+
+  # No network in tests. Answers come from Jevalyn::Testing stubs instead; an
+  # unstubbed evaluation raises rather than silently passing.
+  c.mock_mode = Rails.env.test?
+
+  # Certainty floor a Decision uses when it does not declare its own. Leave nil to
+  # require each Decision to make its own call, which is usually the right default:
+  # the threshold for routing a support ticket is not the threshold for a refund.
+  # c.default_confidence_threshold = 0.75
+end

data/lib/jevalyn/answer.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+module Jevalyn
+  # One typed answer from Jev, wrapped so callers read it as Ruby rather than as a
+  # Hash of strings. Every answer knows the Question it came from, which is what lets
+  # a Choice come back as a Symbol and a Score come back with its label attached.
+  class Answer
+    attr_reader :question, :raw
+
+    def initialize(question, raw)
+      @question = question
+      @raw = raw || {}
+    end
+
+    def name = question.name
+
+    def type = question.type
+
+    # The API's own certainty statistic, 0..1. Choice and Score carry one; a Noul
+    # does not, so this is nil for nouls. Use #certainty for a uniform number.
+    def confidence
+      value = raw["confidence"]
+      value&.to_f
+    end
+
+    # A 0..1 "how sure is this" usable across all three question types.
+    #
+    # For Choice and Score this is the API's own `confidence`. Jev returns no
+    # confidence for a Noul, so Jevalyn derives one from how far the probability sits
+    # from the 0.5 coin-flip: 0.92 and 0.08 are both decisive, 0.5 is not. This is
+    # Jevalyn's arithmetic, not TypeSafe's -- see Answer::Noul#decisiveness.
+    def certainty = confidence
+
+    # True when this answer clears the given confidence floor.
+    def certain?(threshold)
+      return true if threshold.nil?
+
+      value = certainty
+      return false if value.nil?
+
+      value >= threshold
+    end
+
+    def uncertain?(threshold) = !certain?(threshold)
+
+    # The full distribution the answer was derived from, keys left as the API sent them.
+    def probabilities
+      raw["probabilities"] || {}
+    end
+
+    def value = raise(NotImplementedError)
+
+    def to_h = raw
+
+    def inspect
+      "#<#{self.class.name} #{name}=#{value.inspect} certainty=#{certainty.inspect}>"
+    end
+
+    # Probability that a yes/no question is a yes.
+    class Noul < Answer
+      # The raw 0..1 value. Deliberately not rounded to a boolean -- the whole point
+      # of a noul is that you pick the cutoff your use case can afford.
+      def value
+        raw["noul"]&.to_f
+      end
+      alias noul value
+      alias probability value
+
+      # How far from a coin flip the answer sits, rescaled to 0..1.
+      # 0.92 -> 0.84, 0.5 -> 0.0, 0.08 -> 0.84.
+      def decisiveness
+        return nil if value.nil?
+
+        ((value - 0.5).abs * 2).round(10)
+      end
+
+      def certainty = decisiveness
+
+      # Jev returns no confidence for a noul; this stays nil on purpose.
+      def confidence = nil
+
+      def true?(threshold = 0.5)
+        return false if value.nil?
+
+        value >= threshold
+      end
+
+      def false?(threshold = 0.5) = !true?(threshold)
+
+      # A noul has no `probabilities` field; the value is the probability of yes.
+      def probabilities
+        return {} if value.nil?
+
+        { "true" => value, "false" => (1.0 - value).round(10) }
+      end
+    end
+
+    # One option out of the declared set.
+    class Choice < Answer
+      # The winning option, as a Symbol so it reads like the criteria keys you wrote.
+      def value
+        chosen = raw["choice"]
+        chosen&.to_sym
+      end
+      alias choice value
+
+      # The winning option as the API spelled it.
+      def value_s = raw["choice"]
+
+      def probability_of(option)
+        probabilities[option.to_s]&.to_f
+      end
+
+      # Options ordered most to least likely.
+      def ranked
+        probabilities.sort_by { |_, probability| -probability.to_f }
+                     .map { |option, probability| [option.to_sym, probability.to_f] }
+      end
+
+      def runner_up
+        ranked[1]&.first
+      end
+    end
+
+    # A rating against the declared levels.
+    class Score < Answer
+      # The probability-weighted score. A Float, and it lands between levels on
+      # purpose -- 1.6 means "past Frustrated, not quite Very angry".
+      def value
+        raw["score"]&.to_f
+      end
+      alias score value
+
+      # Level index mapped back to its description, as the API returned it.
+      def legend
+        raw["legend"] || {}
+      end
+
+      # The nearest whole level.
+      def level
+        return nil if value.nil?
+
+        value.round
+      end
+
+      # The description of the nearest whole level, e.g. "Very angry".
+      def label
+        return nil if level.nil?
+
+        legend[level.to_s] || question.levels[level]
+      end
+
+      # Highest-probability level, which is not always the nearest to #value.
+      def modal_level
+        top = probabilities.max_by { |_, probability| probability.to_f }
+        top && Integer(top.first)
+      rescue ArgumentError, TypeError
+        nil
+      end
+
+      def probability_of(level_index)
+        probabilities[level_index.to_s]&.to_f
+      end
+
+      # Score normalised to 0..1 across the declared levels, for weighting in code.
+      def normalized
+        return nil if value.nil?
+
+        span = question.levels.length - 1
+        return nil if span <= 0
+
+        (value / span).round(10)
+      end
+    end
+  end
+end

data/lib/jevalyn/client.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "json"
+
+module Jevalyn
+  # Thin, honest wrapper over POST /v1/systemone. It mirrors the HTTP API one-to-one:
+  # a state, a map of questions, one answer per question. Everything ergonomic lives
+  # a layer up in Decision -- this class stays boring on purpose.
+  class Client
+    USER_AGENT = "jevalyn/#{Jevalyn::VERSION} (ruby/#{RUBY_VERSION})".freeze
+
+    # Statuses worth trying again after a backoff. 429 and 529 are the API's own
+    # "slow down" and "come back later"; the rest are transient server faults.
+    RETRYABLE_STATUSES = [429, 500, 502, 503, 504, 529].freeze
+
+    attr_reader :config
+
+    def initialize(config = Jevalyn.config)
+      @config = config
+    end
+
+    # Evaluates a state against a map of questions.
+    #
+    # state     -- String, Hash or Array. Anything responding to #jevalyn_state or
+    #              #as_json is serialised first; see Jevalyn::State.
+    # questions -- Hash of name => Jevalyn::Question, or name => Hash payload.
+    #
+    # thresholds is Jevalyn's own, never sent: a Hash of question => confidence floor,
+    # or a single number for all of them.
+    #
+    # Returns a Jevalyn::Result.
+    def evaluate(state:, questions:, model: nil, confidence_threshold: nil, thresholds: nil,
+                 result_class: Result, decision: nil)
+      questions = normalize_questions(questions)
+      body = {
+        "model" => model || config.default_model,
+        "state" => State.serialize(state),
+        "questions" => questions.each_with_object({}) { |(name, q), out| out[name.to_s] = q.to_payload }
+      }
+
+      raw = if config.mock_mode
+              Testing.answer(body: body, questions: questions, decision: decision)
+            else
+              Testing.through_cassette(body) { post(config.evaluation_url, body) }
+            end
+
+      result_class.new(questions: questions, raw: raw, thresholds: thresholds || confidence_threshold)
+    end
+
+    # GET /v1/models -- the names this account may send in the `model` field.
+    def models
+      return Testing.models if config.mock_mode
+
+      get(config.models_url).fetch("models", [])
+    end
+
+    # Cheap liveness check for `rails runner` or a health endpoint.
+    def reachable?
+      models
+      true
+    end
+
+    private
+
+    def normalize_questions(questions)
+      unless questions.is_a?(Hash) && !questions.empty?
+        raise ConfigurationError,
+              "evaluate needs a non-empty Hash of questions, got #{questions.inspect}."
+      end
+
+      questions.each_with_object({}) do |(name, question), out|
+        out[name.to_sym] =
+          case question
+          when Question then question
+          when Hash     then Question.build(name, **symbolize(question))
+          else
+            raise ConfigurationError,
+                  "Question #{name.inspect} must be a Jevalyn::Question or a Hash, " \
+                  "got #{question.class}."
+          end
+      end
+    end
+
+    def symbolize(hash)
+      hash.each_with_object({}) { |(key, value), out| out[key.to_sym] = value }
+    end
+
+    def post(url, body)
+      request(:post, url) { |req| req.body = JSON.generate(body) }
+    end
+
+    def get(url)
+      request(:get, url)
+    end
+
+    def request(verb, url, &)
+      attempt = 0
+
+      begin
+        response = connection.public_send(verb, url, &)
+        handle(response)
+      rescue Faraday::TimeoutError => e
+        raise TimeoutError, timeout_message(e)
+      rescue Faraday::ConnectionFailed, Faraday::SSLError => e
+        # Faraday reports an open timeout as a connection failure. From the caller's
+        # side it is still a timeout, and the difference matters when deciding whether
+        # to retry, so unwrap it rather than passing the label along.
+        raise TimeoutError, timeout_message(e) if timeout?(e)
+
+        raise ConnectionError, "Could not reach #{config.base_url}: #{e.message}"
+      rescue APIError => e
+        attempt += 1
+        raise unless e.retryable? && attempt <= config.max_retries.to_i
+
+        sleep(backoff_for(attempt, e))
+        retry
+      end
+    end
+
+    def timeout?(error)
+      wrapped = error.respond_to?(:wrapped_exception) ? error.wrapped_exception : nil
+
+      wrapped.is_a?(Timeout::Error) || error.message.to_s.match?(/timed? ?out|execution expired/i)
+    end
+
+    def timeout_message(error)
+      "TypeSafe request timed out after #{config.timeout}s (#{error.message})"
+    end
+
+    def handle(response)
+      body = parse(response.body)
+      return body if response.success?
+
+      raise APIError.from_response(status: response.status, body: body, headers: response.headers)
+    end
+
+    def parse(body)
+      return {} if body.nil? || body.to_s.strip.empty?
+      return body if body.is_a?(Hash) || body.is_a?(Array)
+
+      JSON.parse(body)
+    rescue JSON::ParserError
+      body.to_s
+    end
+
+    # Honour the API's own retry-after when it sends one, exponential backoff otherwise.
+    def backoff_for(attempt, error)
+      requested = error.respond_to?(:retry_after) ? error.retry_after : nil
+      return requested if requested&.positive?
+
+      config.retry_backoff.to_f * (2**(attempt - 1))
+    end
+
+    def connection
+      @connection ||= Faraday.new do |f|
+        f.headers["Authorization"] = "Bearer #{config.api_key!}"
+        f.headers["Content-Type"]  = "application/json"
+        f.headers["Accept"]        = "application/json"
+        f.headers["User-Agent"]    = USER_AGENT
+        f.options.timeout      = config.timeout
+        f.options.open_timeout = config.open_timeout
+        f.adapter Faraday.default_adapter
+      end
+    end
+  end
+end

data/lib/jevalyn/configuration.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Jevalyn
+  # Process-wide settings. Set these once in config/initializers/jevalyn.rb.
+  class Configuration
+    DEFAULT_BASE_URL = "https://api.typesafe.ai"
+    DEFAULT_MODEL    = "jev-latest"
+
+    # The official SDKs default to 10s. Jev answers in well under a second, so a
+    # long timeout only means a slow request holds a Rails thread open.
+    DEFAULT_TIMEOUT      = 10.0
+    DEFAULT_OPEN_TIMEOUT = 5.0
+
+    # 429 and 529 are expected under load; the API asks us to back off, not give up.
+    DEFAULT_MAX_RETRIES   = 2
+    DEFAULT_RETRY_BACKOFF = 0.5
+
+    # Bearer token for the TypeSafe API.
+    attr_accessor :api_key
+
+    # Override to point at a proxy or a recorded fixture server.
+    attr_accessor :base_url
+
+    # Model name or alias sent when a call does not name one.
+    attr_accessor :default_model
+
+    # Per-request read timeout, in seconds.
+    attr_accessor :timeout
+
+    # Connection-open timeout, in seconds.
+    attr_accessor :open_timeout
+
+    # When true, no HTTP request is ever made -- answers come from Jevalyn::Testing.
+    attr_accessor :mock_mode
+
+    # How many times a retryable response (429 / 529 / 5xx) is retried.
+    attr_accessor :max_retries
+
+    # Base delay for exponential backoff between retries, in seconds.
+    attr_accessor :retry_backoff
+
+    # Anything responding to #info / #warn / #debug. Defaults to the Rails logger.
+    attr_accessor :logger
+
+    # Confidence floor a Decision uses when it does not declare its own.
+    attr_accessor :default_confidence_threshold
+
+    # ActiveJob queue used by .evaluate_later.
+    attr_accessor :job_queue_name
+
+    def initialize
+      @api_key       = ENV.fetch("TYPESAFE_API_KEY", nil)
+      @base_url      = ENV.fetch("TYPESAFE_BASE_URL", DEFAULT_BASE_URL)
+      @default_model = ENV.fetch("TYPESAFE_DEFAULT_MODEL", DEFAULT_MODEL)
+      @timeout       = DEFAULT_TIMEOUT
+      @open_timeout  = DEFAULT_OPEN_TIMEOUT
+      @mock_mode     = false
+      @max_retries   = DEFAULT_MAX_RETRIES
+      @retry_backoff = DEFAULT_RETRY_BACKOFF
+      @logger        = nil
+      @default_confidence_threshold = nil
+      @job_queue_name = :default
+    end
+
+    # Raises rather than letting a nil key turn into a confusing 401.
+    def api_key!
+      return @api_key if @api_key && !@api_key.to_s.strip.empty?
+
+      raise ConfigurationError, <<~MSG.strip
+        No TypeSafe API key configured. Set TYPESAFE_API_KEY in the environment, or
+        assign one in config/initializers/jevalyn.rb:
+
+          Jevalyn.configure do |c|
+            c.api_key = ENV["TYPESAFE_API_KEY"]
+          end
+
+        In tests, set `c.mock_mode = true` instead and stub with Jevalyn::Testing.
+      MSG
+    end
+
+    def evaluation_url = "#{base_url.to_s.chomp("/")}/v1/systemone"
+
+    def models_url = "#{base_url.to_s.chomp("/")}/v1/models"
+  end
+end