jevalyn 0.1.0

data/lib/jevalyn/tasks/jevalyn.rake

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+namespace :jevalyn do
+  desc "Check that Jevalyn can reach the TypeSafe API with the configured key"
+  task ping: :environment do
+    config = Jevalyn.config
+    puts "base url: #{config.base_url}"
+    puts "model:    #{config.default_model}"
+    puts "key:      #{config.api_key ? "set (#{config.api_key[0, 6]}...)" : "MISSING"}"
+    puts "mock:     #{config.mock_mode}"
+
+    models = Jevalyn.client.models
+    puts "\nreachable. models available to this account:"
+    models.each { |m| puts "  #{m["name"]} - #{m["description"]}" }
+  rescue Jevalyn::Error => e
+    abort "\n#{e.class}: #{e.message}"
+  end
+
+  desc "List the Decision and Guardrail classes this app defines"
+  task decisions: :environment do
+    Rails.application.eager_load!
+
+    [Jevalyn::Guardrail, Jevalyn::Decision].each do |base|
+      found = base.subclasses.reject { |k| k <= Jevalyn::Guardrail && base == Jevalyn::Decision }
+      next if found.empty?
+
+      puts "#{base.name.demodulize}s:"
+      found.sort_by(&:name).each do |klass|
+        questions = klass.questions.values.map { |q| "#{q.name}:#{q.type}" }.join(" ")
+        puts "  #{klass.name.ljust(32)} #{questions}"
+      end
+      puts
+    end
+  end
+end

data/lib/jevalyn/testing/cassette.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "json"
+require "digest"
+require "fileutils"
+
+module Jevalyn
+  module Testing
+    # Records real evaluations to a JSON file and replays them afterwards, so a suite
+    # pays for each distinct request once instead of on every run.
+    #
+    #   Jevalyn::Testing::Cassette.use("spec/cassettes/triage.json") do
+    #     SupportTriage.evaluate(ticket.body)
+    #   end
+    #
+    # First run with a real API key recorded; every run after that replays. Delete the
+    # file to re-record. Requests are keyed by a digest of the exact body sent, so
+    # changing a rubric misses the cassette rather than silently replaying stale answers.
+    class Cassette
+      attr_reader :path
+
+      def initialize(path)
+        @path = path.to_s
+        @entries = load_entries
+        @recorded = false
+      end
+
+      # Runs the block with this cassette installed.
+      def self.use(path, &)
+        cassette = new(path)
+        cassette.install(&)
+      end
+
+      def install
+        previous = Thread.current[:jevalyn_testing]&.[](:cassette)
+        store[:cassette] = self
+        yield self
+      ensure
+        store[:cassette] = previous
+        save if @recorded
+      end
+
+      # Returns a recorded response for this request body, or nil.
+      def fetch(body)
+        @entries[digest_for(body)]
+      end
+
+      def record(body, response)
+        @entries[digest_for(body)] = response
+        @recorded = true
+        response
+      end
+
+      def recorded? = @recorded
+
+      def size = @entries.size
+
+      def save
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, "#{JSON.pretty_generate(@entries)}\n")
+        @recorded = false
+        path
+      end
+
+      # A stable key for a request. Sorting keys first means a Hash built in a
+      # different order still hits the same entry.
+      def digest_for(body)
+        Digest::SHA256.hexdigest(JSON.generate(deep_sort(body)))[0, 32]
+      end
+
+      private
+
+      def deep_sort(value)
+        case value
+        when Hash  then value.sort_by { |key, _| key.to_s }.to_h { |k, v| [k.to_s, deep_sort(v)] }
+        when Array then value.map { |item| deep_sort(item) }
+        else value
+        end
+      end
+
+      def load_entries
+        return {} unless File.exist?(path)
+
+        JSON.parse(File.read(path))
+      rescue JSON::ParserError => e
+        raise Error, "Cassette #{path} is not valid JSON: #{e.message}"
+      end
+
+      def store
+        Thread.current[:jevalyn_testing] ||= {}
+      end
+    end
+  end
+end

data/lib/jevalyn/testing/minitest.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "jevalyn/testing"
+
+# Minitest integration. In test_helper.rb:
+#
+#   require "jevalyn/testing/minitest"
+#
+#   class ActiveSupport::TestCase
+#     include Jevalyn::Testing::Minitest
+#   end
+module Jevalyn
+  module Testing
+    module Minitest
+      def self.included(base)
+        base.setup do
+          @jevalyn_previous_mock_mode = Jevalyn.config.mock_mode
+          Jevalyn.config.mock_mode = true
+          Jevalyn::Testing.reset!
+        end
+
+        base.teardown do
+          Jevalyn.config.mock_mode = @jevalyn_previous_mock_mode
+          Jevalyn::Testing.reset!
+        end
+      end
+
+      def stub_jevalyn(decision, **values, &)
+        Jevalyn::Testing.stub(decision, **values, &)
+      end
+
+      def assert_evaluated(decision, times: nil)
+        calls = Jevalyn::Testing.calls.select { |call| call.decision == decision }
+
+        if times
+          assert_equal times, calls.length,
+                       "expected #{decision} to be evaluated #{times} time(s), got #{calls.length}"
+        else
+          refute_empty calls, "expected #{decision} to have been evaluated"
+        end
+      end
+
+      def refute_evaluated(decision)
+        calls = Jevalyn::Testing.calls.select { |call| call.decision == decision }
+        assert_empty calls, "expected #{decision} not to have been evaluated"
+      end
+
+      def assert_certain(result, threshold)
+        assert result.certain?(threshold),
+               "expected every answer to clear #{threshold}, but " \
+               "#{result.uncertain_questions(threshold).inspect} did not"
+      end
+    end
+  end
+end

data/lib/jevalyn/testing/rspec.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "jevalyn/testing"
+
+# RSpec integration. Add to spec/spec_helper.rb:
+#
+#   require "jevalyn/testing/rspec"
+#
+# It turns mock_mode on for the suite, clears stubs between examples, and adds:
+#
+#   stub_jevalyn(SupportTriage, department: :technical, urgent: true)
+#   expect(SupportTriage).to have_been_evaluated
+#   expect(result).to be_certain_above(0.8)
+#   expect(result).to have_uncertain_questions(:department)
+#
+# Result's own predicates come through RSpec for free, including the per-question
+# ones a Decision generates:
+#
+#   expect(result).to be_certain
+#   expect(result).to be_department_certain
+#   expect(result).not_to be_severity_certain
+#
+# Tag an example `:jevalyn_live` to let it reach the real API.
+module Jevalyn
+  module Testing
+    module RSpecHelpers
+      def stub_jevalyn(decision, **values, &)
+        Jevalyn::Testing.stub(decision, **values, &)
+      end
+
+      def stub_any_jevalyn(**values, &)
+        Jevalyn::Testing.stub_any(**values, &)
+      end
+
+      def forbid_jevalyn(decision)
+        Jevalyn::Testing.forbid(decision)
+      end
+
+      def jevalyn_calls = Jevalyn::Testing.calls
+
+      # The states passed to a decision, in call order.
+      def jevalyn_states_for(decision)
+        Jevalyn::Testing.calls.select { |call| call.decision == decision }.map(&:state)
+      end
+
+      def jevalyn_cassette(path, &)
+        Jevalyn::Testing::Cassette.use(path, &)
+      end
+    end
+  end
+end
+
+if defined?(RSpec)
+  require "rspec/expectations"
+
+  RSpec::Matchers.define :have_been_evaluated do
+    match do |decision|
+      @calls = Jevalyn::Testing.calls.select { |call| call.decision == decision }
+      @calls = @calls.select { |call| values_match?(@with, call.state) } if defined?(@with)
+      @count ? @calls.length == @count : @calls.any?
+    end
+
+    chain(:times) { |count| @count = count }
+    chain(:once) { @count = 1 }
+    chain(:with_state) { |state| @with = state }
+
+    failure_message do |decision|
+      seen = Jevalyn::Testing.calls.map { |call| call.decision.to_s }.tally
+      "expected #{decision} to have been evaluated#{" #{@count} time(s)" if @count}, " \
+        "but #{seen.empty? ? "nothing was evaluated" : "saw: #{seen.inspect}"}"
+    end
+
+    failure_message_when_negated do |decision|
+      "expected #{decision} not to have been evaluated, but it was #{@calls.length} time(s)"
+    end
+  end
+
+  RSpec::Matchers.define :be_certain_above do |threshold|
+    match { |result| result.certain?(threshold) }
+
+    failure_message do |result|
+      "expected every answer to clear #{threshold}, but " \
+        "#{result.uncertain_questions(threshold).map(&:inspect).join(", ")} did not " \
+        "(lowest certainty #{result.min_certainty.inspect})"
+    end
+  end
+
+  # Asserts exactly which answers missed their own floors. The failure prints the
+  # floor and the certainty side by side, which is the thing you actually need to see
+  # when a per-question threshold is set wrong.
+  RSpec::Matchers.define :have_uncertain_questions do |*expected|
+    match { |result| result.uncertain_questions.sort == expected.flatten.map(&:to_sym).sort }
+
+    failure_message do |result|
+      rows = result.thresholds.map do |name, floor|
+        certainty = result.answer(name).certainty
+        "  #{name}: certainty=#{certainty.inspect} floor=#{floor.inspect}"
+      end
+
+      "expected #{expected.flatten.map(&:to_sym).inspect} to be the uncertain answers, " \
+        "got #{result.uncertain_questions.inspect}\n#{rows.join("\n")}"
+    end
+  end
+
+  RSpec.configure do |config|
+    config.include Jevalyn::Testing::RSpecHelpers
+
+    config.around do |example|
+      previous = Jevalyn.config.mock_mode
+      Jevalyn.config.mock_mode = !example.metadata[:jevalyn_live]
+      Jevalyn::Testing.reset!
+      example.run
+    ensure
+      Jevalyn.config.mock_mode = previous
+      Jevalyn::Testing.reset!
+    end
+  end
+end

data/lib/jevalyn/testing.rb

@@ -0,0 +1,276 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Jevalyn
+  # Keeps test suites off the network.
+  #
+  # TypeSafe has no sandbox key or mock endpoint, so `mock_mode` is entirely local:
+  # with it on, Client never opens a connection and answers come from here instead.
+  #
+  #   Jevalyn::Testing.stub(SupportTriage, department: :technical, urgent: true)
+  #
+  #   result = SupportTriage.evaluate("payouts failing")
+  #   result.department   # => :technical
+  #
+  # Values are written the way you would assert on them -- `true`, `:technical`,
+  # `"major"` -- and Testing expands each into a response the real API could have
+  # returned, probability distribution and all.
+  module Testing
+    # How lopsided a stubbed distribution is. High enough to clear a realistic
+    # confidence threshold without being a fake 1.0 that no real model ever returns.
+    DEFAULT_CONFIDENCE = 0.95
+
+    # What a stubbed `true` and `false` become.
+    DEFAULT_NOUL_TRUE  = 0.95
+    DEFAULT_NOUL_FALSE = 0.05
+
+    Call = Struct.new(:decision, :state, :questions, :model, :raw, keyword_init: true)
+
+    class NoStubError < Error; end
+
+    class << self
+      # Every evaluation made in mock mode, oldest first.
+      def calls
+        store[:calls] ||= []
+      end
+
+      # Registers answers for a Decision.
+      #
+      #   Jevalyn::Testing.stub(SupportTriage, urgent: true, department: :technical)
+      #   Jevalyn::Testing.stub(SupportTriage, confidence: 0.6, department: :billing)
+      #   Jevalyn::Testing.stub(SupportTriage) { |state| { department: route_for(state) } }
+      #
+      # `confidence:` also takes a Hash, which is how you exercise per-question floors:
+      # one answer landing under its floor while another clears its own.
+      #
+      #   Jevalyn::Testing.stub(SupportTriage,
+      #     department: :technical, severity: "major",
+      #     confidence: { department: 0.7, severity: 0.65 })
+      #
+      # A block is re-run per call and receives the serialised state, so one stub can
+      # answer differently for different inputs.
+      def stub(decision, confidence: DEFAULT_CONFIDENCE, **values, &block)
+        stubs[key_for(decision)] = { values: values, confidence: confidence, block: block }
+        decision
+      end
+
+      # A catch-all for decisions with no stub of their own.
+      def stub_any(confidence: DEFAULT_CONFIDENCE, **values, &block)
+        stubs[:__any__] = { values: values, confidence: confidence, block: block }
+      end
+
+      # Raises instead of answering, to prove a code path does not call Jev.
+      def forbid(decision)
+        stubs[key_for(decision)] = :forbidden
+      end
+
+      def stubbed?(decision) = stubs.key?(key_for(decision))
+
+      def reset!
+        store[:stubs] = {}
+        store[:calls] = []
+        store[:cassette] = nil
+      end
+
+      # Called by Client when config.mock_mode is on. Returns a raw response Hash.
+      def answer(body:, questions:, decision: nil)
+        raw = cassette&.fetch(body) || build_response(body: body, questions: questions, decision: decision)
+
+        calls << Call.new(
+          decision: decision, state: body["state"], questions: questions,
+          model: body["model"], raw: raw
+        )
+
+        raw
+      end
+
+      # Wraps a real API call in whatever cassette is currently installed: replays a
+      # recorded response when there is one, records the live answer when there is not.
+      def through_cassette(body)
+        active = cassette
+        return yield unless active
+
+        active.fetch(body) || active.record(body, yield)
+      end
+
+      # Stand-in for GET /v1/models.
+      def models
+        [
+          { "name" => "jev-latest",  "description" => "Most recent stable release" },
+          { "name" => "jev-preview", "description" => "Most recent release, stable or not" }
+        ]
+      end
+
+      # Turns friendly values into a response body the API could have produced.
+      def build_response(body:, questions:, decision: nil)
+        entry = stub_for(decision)
+
+        if entry == :forbidden
+          raise NoStubError,
+                "#{decision} is forbidden in this example, but something evaluated it."
+        end
+
+        values, confidence = resolve(entry, body)
+
+        answers = questions.each_with_object({}) do |(name, question), out|
+          raise NoStubError, missing_message(decision, name, questions) unless values.key?(name)
+
+          out[name.to_s] = answer_for(question, values[name], confidence_for(confidence, name))
+        end
+
+        {
+          "model" => body["model"] == "jev-latest" ? "jev-1.13.0" : body["model"],
+          "answers" => answers,
+          "usage" => { "input_tokens" => State.estimated_tokens(body["state"]), "output_tokens" => 0 }
+        }
+      end
+
+      # Builds one answer object of the right shape for a question type.
+      def answer_for(question, value, confidence)
+        case question.type
+        when :noul   then noul_answer(value)
+        when :choice then choice_answer(question, value, confidence)
+        when :score  then score_answer(question, value, confidence)
+        end
+      end
+
+      private
+
+      # `confidence:` is either one number for every answer, or a Hash naming them
+      # individually so a spec can put one answer under its floor and another over.
+      def confidence_for(confidence, name)
+        return confidence unless confidence.is_a?(Hash)
+
+        confidence[name] || confidence[name.to_s] || DEFAULT_CONFIDENCE
+      end
+
+      def noul_answer(value)
+        probability =
+          case value
+          when true     then DEFAULT_NOUL_TRUE
+          when false    then DEFAULT_NOUL_FALSE
+          when Numeric  then value.to_f
+          else
+            raise ArgumentError,
+                  "A noul stub takes true, false or a Float between 0 and 1, got #{value.inspect}."
+          end
+
+        { "type" => "noul", "noul" => probability.round(6) }
+      end
+
+      def choice_answer(question, value, confidence)
+        chosen = value.to_s
+        options = question.criteria.keys.map(&:to_s)
+
+        unless options.include?(chosen)
+          raise ArgumentError,
+                "#{chosen.inspect} is not an option of :#{question.name}. " \
+                "It accepts: #{options.map(&:inspect).join(", ")}."
+        end
+
+        {
+          "type" => "choice",
+          "choice" => chosen,
+          "probabilities" => distribute(options, chosen, confidence),
+          "confidence" => confidence.to_f.round(6)
+        }
+      end
+
+      def score_answer(question, value, confidence)
+        levels = question.levels
+        index =
+          case value
+          when Integer then value
+          when Float   then value.round
+          when String, Symbol
+            found = levels.index(value.to_s)
+            unless found
+              raise ArgumentError,
+                    "#{value.inspect} is not a level of :#{question.name}. " \
+                    "It has: #{levels.map(&:inspect).join(", ")}."
+            end
+            found
+          else
+            raise ArgumentError,
+                  "A score stub takes a level name, or its index, got #{value.inspect}."
+          end
+
+        unless index.between?(0, levels.length - 1)
+          raise ArgumentError,
+                "Level #{index} is out of range for :#{question.name} (0..#{levels.length - 1})."
+        end
+
+        keys = (0...levels.length).map(&:to_s)
+        probabilities = distribute(keys, index.to_s, confidence)
+        # The real API returns a probability-weighted score, so do the same arithmetic
+        # here -- a stub that always returns a whole number hides rounding bugs.
+        weighted = probabilities.sum { |level, probability| level.to_i * probability }
+
+        {
+          "type" => "score",
+          "score" => weighted.round(6),
+          "legend" => keys.zip(levels).to_h,
+          "probabilities" => probabilities,
+          "confidence" => confidence.to_f.round(6)
+        }
+      end
+
+      # Puts `confidence` of the mass on the winner and spreads the rest evenly.
+      def distribute(keys, winner, confidence)
+        confidence = confidence.to_f.clamp(0.0, 1.0)
+        others = keys - [winner]
+        remainder = others.empty? ? 0.0 : (1.0 - confidence) / others.length
+
+        keys.to_h { |key| [key, (key == winner ? confidence : remainder).round(6)] }
+      end
+
+      def resolve(entry, body)
+        unless entry
+          raise NoStubError, <<~MSG.strip
+            Jevalyn is in mock_mode and nothing is stubbed for this evaluation.
+
+              Jevalyn::Testing.stub(YourDecision, question_name: value)
+
+            Or allow real calls in this example with `Jevalyn.config.mock_mode = false`.
+          MSG
+        end
+
+        values = entry[:values] || {}
+        values = values.merge(entry[:block].call(body["state"]) || {}) if entry[:block]
+        [symbolize(values), entry[:confidence] || DEFAULT_CONFIDENCE]
+      end
+
+      def stub_for(decision)
+        stubs[key_for(decision)] || stubs[:__any__]
+      end
+
+      def missing_message(decision, name, questions)
+        <<~MSG.strip
+          No stubbed value for :#{name} on #{decision || "this evaluation"}.
+
+            Jevalyn::Testing.stub(#{decision || "YourDecision"}, #{questions.keys.map { |k| "#{k}: ..." }.join(", ")})
+        MSG
+      end
+
+      def symbolize(hash)
+        hash.each_with_object({}) { |(key, value), out| out[key.to_sym] = value }
+      end
+
+      def key_for(decision)
+        decision.is_a?(Class) ? decision.name || decision.object_id : decision.to_s
+      end
+
+      def cassette = store[:cassette]
+
+      def stubs
+        store[:stubs] ||= {}
+      end
+
+      # Thread-local so parallel specs do not stub over each other.
+      def store
+        Thread.current[:jevalyn_testing] ||= {}
+      end
+    end
+  end
+end

data/lib/jevalyn/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Jevalyn
+  VERSION = "0.1.0"
+end

data/lib/jevalyn.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "jevalyn/version"
+require "jevalyn/errors"
+require "jevalyn/configuration"
+require "jevalyn/question"
+require "jevalyn/answer"
+require "jevalyn/result"
+require "jevalyn/state_adapters/active_record_adapter"
+require "jevalyn/state"
+require "jevalyn/testing"
+require "jevalyn/testing/cassette"
+require "jevalyn/client"
+require "jevalyn/decision"
+require "jevalyn/guardrail"
+require "jevalyn/router"
+require "jevalyn/evaluation_job"
+
+require "jevalyn/railtie" if defined?(Rails::Railtie)
+
+# Jevalyn is the decision layer for your Rails app.
+#
+# It wraps TypeSafe's Jev, a System One model: you give it a state and a set of typed
+# questions, and it gives back typed, calibrated answers. A probability that something
+# is true, one category out of a set you defined, a rating against your own rubric.
+#
+# That is the whole surface. Jev does not write prose, summarise, or reason
+# open-endedly, and Jevalyn does not pretend otherwise. What it does is make a
+# decision cheap enough and fast enough to sit directly in a Rails request.
+#
+#   class SupportTriage < Jevalyn::Decision
+#     question :department, type: :choice,
+#       instructions: "Which team should handle this?",
+#       criteria: {
+#         billing:   "Payments, invoicing, refunds",
+#         technical: "Bugs, outages, integrations",
+#         sales:     "Pricing, upgrades, new accounts"
+#       }
+#   end
+#
+#   SupportTriage.evaluate(ticket.body).department   # => :technical
+module Jevalyn
+  class << self
+    def config
+      @config ||= Configuration.new
+    end
+
+    # Set up in config/initializers/jevalyn.rb.
+    #
+    #   Jevalyn.configure do |c|
+    #     c.api_key = ENV["TYPESAFE_API_KEY"]
+    #   end
+    def configure
+      yield config
+      reset_client!
+      config
+    end
+
+    # The shared Client. Thread-safe to read; rebuilt whenever config changes.
+    def client
+      @client ||= Client.new(config)
+    end
+
+    def logger = config.logger
+
+    # One-off evaluation without declaring a Decision class. Fine in a console or a
+    # rake task; in application code a Decision gives you validation and a name.
+    #
+    #   Jevalyn.evaluate(
+    #     "Help! My payouts have been failing for 3 days.",
+    #     urgent: { type: :noul, instructions: "Does this convey urgency?" }
+    #   )
+    def evaluate(state, **questions)
+      client.evaluate(state: state, questions: questions)
+    end
+
+    def reset_client!
+      @client = nil
+    end
+
+    # Resets everything, for specs that mutate configuration.
+    def reset!
+      @config = nil
+      @client = nil
+      Testing.reset!
+    end
+  end
+end