judge_rails 0.0.1

data/lib/judge/rails/storage.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Judge
+  module Rails
+    module Storage
+      module_function
+
+      def plan(record, definitions, force: false)
+        definitions.each_with_object([{}, []]) do |definition, (pending, blank)|
+          next unless loaded?(record, definition) && definition.applies_to?(record)
+
+          state = readable_state(record, definition)
+          next if state.nil?
+
+          if state.empty?
+            blank << definition unless cleared?(record, definition)
+          elsif force || stale?(record, definition, state)
+            pending[definition] = state
+          end
+        end
+      end
+
+      def pending(record, definitions, force: false)
+        plan(record, definitions, force: force).first
+      end
+
+      def stale_definitions(record, definitions, force: false)
+        pending, blank = plan(record, definitions, force: force)
+        pending.keys + blank
+      end
+
+      def groups(pending)
+        pending.group_by { |definition, state| [state, definition.effective_model] }
+               .transform_values { |pairs| pairs.map(&:first) }
+      end
+
+      def compute(record, definitions, force: false, adapter: nil, done: [])
+        pending, blank = plan(record, definitions, force: force)
+        blank.each do |definition|
+          clear(record, definition)
+          done << definition
+        end
+        groups(pending).each do |(state, model), group|
+          yield if block_given?
+          ask(record, state, model, group, adapter: adapter)
+          done.concat(group)
+        end
+        done
+      end
+
+      def ask(record, state, model, group, adapter: nil)
+        questions = group.to_h { |d| [d.name, d.question] }
+        results = Judge.ask(questions, text: state, model: model, adapter: adapter)
+        group.each { |d| write(record, d, results[d.name], state: state, results: results) }
+      end
+
+      def write(record, definition, result, state:, results: nil)
+        raise Judge::InvalidResponseError, "no answer for #{definition.name.inspect}" if result.nil?
+
+        record.write_attribute(definition.value_column, definition.cast(result))
+        record.write_attribute(
+          definition.sidecar_column,
+          definition.sidecar(result, state_digest: definition.state_digest(state),
+                                     model: results&.model, latency: results&.latency)
+        )
+      end
+
+      def clear(record, definition)
+        record.write_attribute(definition.value_column, nil)
+        record.write_attribute(definition.sidecar_column, {})
+      end
+
+      def sidecar(record, definition)
+        value = record.read_attribute(definition.sidecar_column)
+        case value
+        when Hash then value.dup
+        when String then value.empty? ? {} : JSON.parse(value)
+        else {}
+        end
+      end
+
+      def stale?(record, definition, state)
+        definition.stale?(value: record.read_attribute(definition.value_column),
+                          sidecar: sidecar(record, definition), state: state)
+      end
+
+      def cleared?(record, definition)
+        record.read_attribute(definition.value_column).nil? && sidecar(record, definition).empty?
+      end
+
+      def loaded?(record, definition)
+        [definition.value_column, definition.sidecar_column].all? do |column|
+          record.has_attribute?(column) || !record.class.has_attribute?(column)
+        end
+      end
+
+      def readable_state(record, definition)
+        definition.state_for(record)
+      rescue ActiveModel::MissingAttributeError
+        nil
+      end
+    end
+  end
+end

data/lib/judge/rails/validator.rb

@@ -0,0 +1,246 @@
+# frozen_string_literal: true
+
+I18n.load_path << File.expand_path("locale/en.yml", __dir__)
+
+module Judge
+  module Rails
+    module Validator
+      extend ActiveSupport::Concern
+
+      def judge_validation_results
+        @judge_validation_results ||= {}
+      end
+
+      def judge_judgment(text, instruction)
+        @judge_judgments && @judge_judgments[[text, instruction]]
+      end
+
+      def judge_store_judgment(text, instruction, outcome)
+        @judge_judgments ||= {}
+        @judge_judgments[[text, instruction]] = outcome
+      end
+
+      def judge_record_validation_result(attribute, instruction, result)
+        (judge_validation_results[attribute] ||= []) << [instruction, result]
+      end
+
+      def judge_prefetch
+        return if @judge_prefetched
+
+        @judge_prefetched = true
+        ::JudgeValidator.prefetch(self)
+      end
+
+      def judge_saw_text(text)
+        (@judge_seen_texts ||= Set.new) << text
+      end
+
+      def reload(*)
+        @judge_judgments = nil
+        super
+      end
+
+      def initialize_dup(other)
+        super
+        @judge_judgments = nil
+        @judge_validation_results = nil
+      end
+
+      private
+
+      def run_validations!
+        @judge_validation_results = {}
+        @judge_prefetched = false
+        @judge_seen_texts = nil
+        super
+      ensure
+        judge_prune_judgments
+      end
+
+      def judge_prune_judgments
+        return unless @judge_judgments
+
+        seen = @judge_seen_texts || Set.new
+        @judge_judgments.select! { |(text, _), (kind, _)| kind == :ok && seen.include?(text) }
+      end
+    end
+  end
+end
+
+class JudgeValidator < ActiveModel::EachValidator # rubocop:disable Style/OneClassPerFile
+  ERROR_MODES = %i[pass fail raise].freeze
+
+  attr_reader :instruction, :polarity, :threshold, :question, :on_error
+
+  def self.prefetch(record)
+    groups = Hash.new { |hash, key| hash[key] = [] }
+
+    record.class.validators.each do |validator|
+      next unless validator.is_a?(JudgeValidator) && validator.applicable?(record)
+
+      validator.attributes.each do |attribute|
+        value = record.read_attribute_for_validation(attribute)
+        next if value.blank? || record.judge_judgment(value.to_s, validator.instruction)
+
+        group = groups[value.to_s]
+        group << validator unless group.any? { |other| other.instruction == validator.instruction }
+      end
+    end
+
+    groups.each { |text, validators| ask_batch(record, text, validators) if validators.size > 1 }
+  end
+
+  def self.ask_batch(record, text, validators)
+    questions = validators.each_with_index.to_h { |validator, i| [:"judge_#{i}", validator.question] }
+    results = Judge.ask(questions, text: text)
+    validators.each_with_index do |validator, i|
+      result = results[:"judge_#{i}"]
+      raise Judge::InvalidResponseError, "no answer for #{validator.instruction}" unless result
+
+      record.judge_store_judgment(text, validator.instruction, [:ok, result])
+    end
+  rescue Judge::Error => e
+    validators.each { |validator| record.judge_store_judgment(text, validator.instruction, [:error, e]) }
+  end
+
+  def initialize(options)
+    @polarity, @instruction = extract_judgment(options)
+    @threshold = Float(options.fetch(:threshold, 0.5))
+    @on_error = (options[:on_error] || :pass).to_sym
+    unless ERROR_MODES.include?(@on_error)
+      raise ArgumentError, "judge on_error: must be one of #{ERROR_MODES.inspect}"
+    end
+
+    @question = Judge.noul(@instruction)
+    super
+  end
+
+  def validate_each(record, attribute, value)
+    return if value.blank?
+
+    kind, payload = judgment(record, attribute, value.to_s)
+    if kind == :error
+      handle_error(record, attribute, payload)
+    else
+      if record.respond_to?(:judge_record_validation_result)
+        record.judge_record_validation_result(attribute, instruction, payload)
+      end
+      add_error(record, attribute, violation_type, options[:message]) if violated?(payload)
+    end
+  end
+
+  def applicable?(record)
+    return false unless context_match?(record)
+
+    Array(options[:if]).all? { |condition| evaluate_condition(record, condition) } &&
+      Array(options[:unless]).none? { |condition| evaluate_condition(record, condition) }
+  end
+
+  private
+
+  def extract_judgment(options)
+    refute = options[:refute]
+    assert = options[:assert]
+    if (refute && assert) || (refute.nil? && assert.nil?)
+      raise ArgumentError, "judge validation needs exactly one of refute: or assert:"
+    end
+
+    refute ? [:refute, refute.to_s] : [:assert, assert.to_s]
+  end
+
+  def judgment(record, attribute, text)
+    record.judge_prefetch if record.respond_to?(:judge_prefetch)
+    record.judge_saw_text(text) if record.respond_to?(:judge_saw_text)
+    cached = cached_judgment(record, attribute, text)
+    return cached if cached
+
+    outcome = ask(text)
+    store_judgment(record, attribute, text, outcome)
+    outcome
+  end
+
+  def cached_judgment(record, attribute, text)
+    return record.judge_judgment(text, instruction) if record.respond_to?(:judge_judgment)
+
+    cached_text, outcome = fallback_cache(record)&.[]([attribute, instruction])
+    outcome if cached_text == text
+  end
+
+  def store_judgment(record, attribute, text, outcome)
+    if record.respond_to?(:judge_store_judgment)
+      record.judge_store_judgment(text, instruction, outcome)
+    elsif outcome.first == :ok
+      fallback_cache(record)&.[]=([attribute, instruction], [text, outcome])
+    end
+  end
+
+  def fallback_cache(record)
+    return record.instance_variable_get(:@judge_judgments) if record.frozen?
+
+    record.instance_variable_get(:@judge_judgments) || record.instance_variable_set(:@judge_judgments, {})
+  end
+
+  def ask(text)
+    [:ok, Judge.ask(question, text: text)]
+  rescue Judge::Error => e
+    [:error, e]
+  end
+
+  def violated?(result)
+    polarity == :refute ? result.true?(threshold) : !result.true?(threshold)
+  end
+
+  def violation_type
+    polarity == :refute ? :judge_refuted : :judge_unmatched
+  end
+
+  def handle_error(record, attribute, error)
+    case on_error
+    when :fail then add_error(record, attribute, :judge_unavailable, nil)
+    when :raise then raise error
+    else log_skipped(record, attribute, error)
+    end
+  end
+
+  def log_skipped(record, attribute, error)
+    Judge::Rails.logger&.warn("[judge] #{record.class} #{attribute} not checked: " \
+                              "#{error.class}: #{error.message}")
+  end
+
+  def add_error(record, attribute, type, message)
+    details = { instruction: instruction }
+    message ||= english_fallback(type)
+    details[:message] = message if message
+    details[:strict] = options[:strict] if options[:strict]
+    record.errors.add(attribute, type, **details)
+  end
+
+  def english_fallback(type)
+    key = :"errors.messages.#{type}"
+    return if I18n.exists?(key)
+
+    I18n.t(key, locale: :en, instruction: instruction)
+  end
+
+  def context_match?(record)
+    contexts = Array(record.send(:validation_context)).map(&:to_s)
+    on = options[:on]
+    except_on = options[:except_on]
+    return false if on && !Array(on).map(&:to_s).intersect?(contexts)
+    return false if except_on && Array(except_on).map(&:to_s).intersect?(contexts)
+
+    true
+  end
+
+  def run_proc(record, condition)
+    condition.arity.zero? ? record.instance_exec(&condition) : record.instance_exec(record, &condition)
+  end
+
+  def evaluate_condition(record, condition)
+    case condition
+    when Symbol, String then record.send(condition)
+    when Proc then run_proc(record, condition)
+    else condition.respond_to?(:validate) ? condition.validate(record) : condition.call(record)
+    end
+  end
+end

data/lib/judge/rails.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "judge"
+require "active_support"
+require "active_record"
+require "judge/rails/definition"
+require "judge/rails/registry"
+require "judge/rails/storage"
+require "judge/rails/attributes"
+require "judge/rails/migration"
+require "judge/rails/refresh"
+require "judge/rails/jobs"
+require "judge/rails/scopes"
+require "judge/rails/relation"
+require "judge/rails/validator"
+
+module Judge
+  module Rails
+    CONCERNS = [Attributes, Refresh, Jobs, Scopes, Validator].freeze
+    LEASE_REGISTRY_LOCK = Mutex.new
+
+    def self.logger
+      Judge.config.logger || ActiveRecord::Base.logger
+    end
+
+    def self.log_failure(context, error)
+      logger&.error("[judge] refresh failed for #{context}: #{error.class}: #{error.message}")
+    end
+  end
+end
+
+Judge::Pool.on_worker_exit do
+  Judge::Rails::LEASE_REGISTRY_LOCK.synchronize do
+    ActiveRecord::Base.connection_handler.connection_pool_list(:all).each do |pool|
+      pool.release_connection if pool.active_connection?
+    end
+  end
+end
+
+ActiveSupport.on_load(:active_record) do
+  Judge::Rails::CONCERNS.each { |mod| include mod }
+  extend Judge::Rails::Relation::ClassMethods
+end
+
+ActiveSupport.on_load(:active_job) { require "judge/rails/refresh_job" }

data/lib/judge/result.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+module Judge
+  class Result
+    UNSURE = :unsure
+
+    TYPES = %w[noul choice score].freeze
+
+    attr_reader :name, :question, :raw, :type, :value, :probability
+
+    def self.from_values(name:, type:, value:, question: nil, confidence: nil,
+                         probabilities: nil, legend: nil)
+      kind = type.to_s
+      payload = { "type" => kind, kind => value }
+      payload["confidence"] = confidence unless confidence.nil?
+      payload["probabilities"] = probabilities.to_h { |k, v| [k.to_s, v] } if probabilities
+      payload["legend"] = legend.to_h { |k, v| [k.to_s, v] } if legend
+      new(name: name, question: question, payload: payload)
+    end
+
+    def self.frozen_copy(value)
+      case value
+      when Hash then value.to_h { |k, v| [frozen_copy(k), frozen_copy(v)] }.freeze
+      when Array then value.map { |v| frozen_copy(v) }.freeze
+      when String then value.dup.freeze
+      else value
+      end
+    end
+
+    def initialize(name:, question:, payload:)
+      unless payload.is_a?(Hash)
+        raise InvalidResponseError, "expected an answer object for #{name.inspect}, got #{payload.class}"
+      end
+
+      @name = name&.to_sym
+      @question = question
+      @raw = self.class.frozen_copy(payload)
+      @type = resolve_type
+      check_shape!
+      @value = parse_value
+      @probability = compute_probability
+      freeze
+    end
+
+    def confidence
+      raw["confidence"]
+    end
+
+    def probabilities
+      raw["probabilities"] || {}
+    end
+
+    def legend
+      raw["legend"] || {}
+    end
+
+    def level
+      return unless type == "score"
+
+      value.round
+    end
+
+    def label
+      return unless type == "score"
+
+      legend[level.to_s]
+    end
+
+    def probability_of(option)
+      probabilities[option.to_s]
+    end
+
+    def true?(threshold = 0.5)
+      unless type == "noul"
+        raise ArgumentError,
+              "true? is only meaningful for noul answers, this is a #{type}"
+      end
+
+      value >= threshold
+    end
+
+    def confident?(threshold = 0.8)
+      (confidence || probability || 0) >= threshold
+    end
+
+    def decide(above:, below: nil)
+      raise ArgumentError, "decide needs a numeric probability" if probability.nil?
+
+      Decision.call(probability, above: above, below: below)
+    end
+
+    def to_h
+      { name: name, type: type, value: value, probability: probability, confidence: confidence }
+    end
+
+    def inspect
+      "#<Judge::Result #{name.inspect} #{type} value=#{value.inspect} p=#{probability.inspect}>"
+    end
+
+    private
+
+    def resolve_type
+      kind = raw["type"] || question&.type
+      unless TYPES.include?(kind)
+        raise InvalidResponseError,
+              "unknown answer type #{kind.inspect} for #{name.inspect}"
+      end
+      if question && question.type != kind
+        raise InvalidResponseError, "a #{question.type} question got a #{kind} answer for #{name.inspect}"
+      end
+
+      kind
+    end
+
+    def check_shape!
+      unless raw["probabilities"].nil? || raw["probabilities"].is_a?(Hash)
+        raise InvalidResponseError, "probabilities for #{name.inspect} must be an object"
+      end
+      unless raw["legend"].nil? || raw["legend"].is_a?(Hash)
+        raise InvalidResponseError, "legend for #{name.inspect} must be an object"
+      end
+      return if raw["confidence"].nil? || raw["confidence"].is_a?(Numeric)
+
+      raise InvalidResponseError, "confidence for #{name.inspect} must be a number"
+    end
+
+    def parse_value
+      case type
+      when "noul" then number("noul")
+      when "choice" then choice_value
+      when "score" then score_value
+      end
+    end
+
+    def compute_probability
+      case type
+      when "noul" then value
+      when "choice" then probabilities[value]
+      when "score" then probabilities[level.to_s]
+      end
+    end
+
+    def choice_value
+      choice = raw["choice"]
+      raise InvalidResponseError, "missing choice in answer #{name.inspect}" unless choice.is_a?(String)
+      if question.respond_to?(:options) && !question.options.include?(choice)
+        raise InvalidResponseError, "#{choice.inspect} is not an option of #{name.inspect}"
+      end
+
+      choice
+    end
+
+    def score_value
+      score = number("score")
+      if question.respond_to?(:max_level) && !score.between?(0, question.max_level)
+        raise InvalidResponseError, "score #{score} for #{name.inspect} is outside 0..#{question.max_level}"
+      end
+
+      score
+    end
+
+    def number(key)
+      v = raw[key]
+      raise InvalidResponseError, "missing #{key} in answer #{name.inspect}" if v.nil?
+      unless v.is_a?(Numeric)
+        raise InvalidResponseError,
+              "#{key} in answer #{name.inspect} is not a number: #{v.inspect}"
+      end
+
+      Float(v)
+    end
+  end
+end

data/lib/judge/result_set.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Judge
+  class ResultSet
+    include Enumerable
+
+    Usage = Struct.new(:input_tokens, :output_tokens) do
+      def total
+        input_tokens.to_i + output_tokens.to_i
+      end
+    end
+
+    attr_reader :model, :usage, :latency, :raw
+
+    def initialize(results, model: nil, usage: nil, latency: nil, raw: nil)
+      @results = results.to_h { |r| [r.name, r] }.freeze
+      raise ArgumentError, "duplicate result names" if @results.size != results.size
+
+      @model = model
+      @usage = usage
+      @latency = latency
+      @raw = raw
+      freeze
+    end
+
+    def self.from_response(body, questions:, latency: nil)
+      answers = body["answers"]
+      raise InvalidResponseError, "response has no answers" unless answers.is_a?(Hash)
+
+      results = questions.map do |name, question|
+        payload = answers[name.to_s]
+        raise InvalidResponseError, "no answer for #{name.inspect}" if payload.nil?
+
+        question.coerce(payload, name: name)
+      end
+
+      usage = parse_usage(body["usage"])
+      new(results, model: body["model"], usage: usage, latency: latency, raw: body)
+    end
+
+    def self.parse_usage(raw)
+      return unless raw.is_a?(Hash)
+
+      Usage.new(raw["input_tokens"], raw["output_tokens"]).freeze
+    end
+
+    def [](name)
+      @results[name.to_sym]
+    end
+
+    def fetch(name)
+      @results.fetch(name.to_sym)
+    end
+
+    def each(&)
+      @results.each_value(&)
+    end
+
+    def names
+      @results.keys
+    end
+
+    def size
+      @results.size
+    end
+
+    def to_h
+      @results.transform_values(&:to_h)
+    end
+
+    def inspect
+      "#<Judge::ResultSet #{@results.keys.inspect} model=#{model.inspect} latency=#{latency&.round(3)}>"
+    end
+  end
+end

data/lib/judge/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Judge
+  VERSION = "0.0.1"
+end

data/lib/judge.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "judge/version"
+require "judge/errors"
+require "judge/configuration"
+require "judge/question"
+require "judge/question/noul"
+require "judge/question/choice"
+require "judge/question/score"
+require "judge/result"
+require "judge/result_set"
+require "judge/decision"
+require "judge/facade"
+require "judge/adapter"
+require "judge/pool"
+
+module Judge
+  extend Facade
+
+  autoload :Client, "judge/client"
+
+  @config_lock = Mutex.new
+
+  class << self
+    def config
+      @config || @config_lock.synchronize { @config ||= Configuration.new }
+    end
+
+    def configure
+      yield config
+      config
+    end
+
+    def reset_config!
+      @config = Configuration.new
+      self.adapter = nil
+      Adapter.reset_built!
+      config
+    end
+  end
+end

data/lib/judge_rails.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require "judge/rails"