judge_rails 0.0.1

data/lib/judge/facade.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Judge
+  module Facade
+    DEFAULT_NAME = :answer
+
+    def noul(instructions, criteria = nil, name: nil)
+      Question::Noul.new(instructions, criteria, name: name)
+    end
+
+    def choice(instructions, options, name: nil)
+      Question::Choice.new(instructions, options, name: name)
+    end
+
+    def score(instructions, levels, name: nil)
+      Question::Score.new(instructions, levels, name: name)
+    end
+
+    def ask(questions, text:, model: nil, adapter: nil)
+      single = single_question?(questions)
+      normalized = normalize_questions(questions)
+      raise ArgumentError, "ask needs at least one question" if normalized.empty?
+
+      results = (adapter || self.adapter).call(state: text, questions: normalized, model: model)
+      return results unless single
+
+      name = normalized.keys.first
+      results[name] || raise(InvalidResponseError, "no answer for #{name.inspect}")
+    end
+
+    def adapter
+      @adapter || Adapter.resolve(config.adapter)
+    end
+
+    attr_writer :adapter
+
+    private
+
+    def normalize_questions(input)
+      case input
+      when Question then { input.name || DEFAULT_NAME => input }
+      when String then { DEFAULT_NAME => noul(input) }
+      when Hash then named_from_hash(input)
+      when Array then named_from_array(input)
+      else raise ArgumentError, "expected a Question, String, Array or Hash, got #{input.class}"
+      end
+    end
+
+    def named_from_hash(input)
+      named = input.to_h { |name, q| [name.to_sym, coerce_question(q).with_name(name)] }
+      raise ArgumentError, "duplicate question names" if named.size != input.size
+
+      named
+    end
+
+    def single_question?(input)
+      input.is_a?(Question) || input.is_a?(String)
+    end
+
+    def coerce_question(value)
+      value.is_a?(String) ? noul(value) : value
+    end
+
+    def named_from_array(list)
+      named = list.each_with_index.to_h do |q, i|
+        question = coerce_question(q)
+        name = question.name || :"q#{i}"
+        [name, question.with_name(name)]
+      end
+      raise ArgumentError, "duplicate question names" if named.size != list.size
+
+      named
+    end
+  end
+end

data/lib/judge/pool.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Judge
+  module Pool
+    DEFAULT_CONCURRENCY = 8
+    THREAD_NAME = "judge-pool"
+
+    @worker_exit_hooks = []
+
+    class << self
+      def map(items, concurrency: nil, &work)
+        list = items.to_a
+        return [] if list.empty?
+
+        workers = (concurrency || Judge.config.concurrency || DEFAULT_CONCURRENCY).to_i
+        return list.map(&work) if workers <= 1 || list.one?
+
+        fan_out(list, [workers, list.size].min, &work)
+      end
+
+      def on_worker_exit(&hook)
+        @worker_exit_hooks << hook unless @worker_exit_hooks.include?(hook)
+        hook
+      end
+
+      private
+
+      def fan_out(list, workers, &work)
+        queue = Queue.new
+        list.each_with_index { |item, index| queue << [index, item] }
+        results = Array.new(list.size)
+        errors = []
+        lock = Mutex.new
+
+        threads = Array.new(workers) do
+          spawn(queue) do
+            while (job = pop(queue))
+              index, item = job
+              begin
+                value = work.call(item)
+                lock.synchronize { results[index] = value }
+              rescue StandardError => e
+                lock.synchronize { errors << e }
+                drain(queue)
+              end
+            end
+          end
+        end
+        threads.each(&:join)
+
+        raise errors.first unless errors.empty?
+
+        results
+      ensure
+        drain(queue) if queue
+      end
+
+      def spawn(queue, &body)
+        tags = log_tags
+        thread = Thread.new do
+          finished = false
+          tagged(tags, &body)
+          finished = true
+        ensure
+          drain(queue) unless finished
+          @worker_exit_hooks.each(&:call)
+        end
+        thread.name = THREAD_NAME
+        thread.report_on_exception = false
+        thread
+      end
+
+      def log_tags
+        logger = Judge.config.logger
+        formatter = logger.respond_to?(:formatter) && logger.formatter
+        formatter.respond_to?(:current_tags) ? formatter.current_tags.dup : []
+      end
+
+      def tagged(tags, &)
+        logger = Judge.config.logger
+        tags.empty? || !logger.respond_to?(:tagged) ? yield : logger.tagged(*tags, &)
+      end
+
+      def pop(queue)
+        queue.pop(true)
+      rescue ThreadError
+        nil
+      end
+
+      def drain(queue)
+        loop { queue.pop(true) }
+      rescue ThreadError
+        nil
+      end
+    end
+  end
+end

data/lib/judge/question/choice.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Judge
+  class Question
+    class Choice < Question
+      TYPE = "choice"
+
+      def options
+        criteria.keys
+      end
+
+      private
+
+      def normalize(criteria)
+        options = case criteria
+                  when Hash then entries(criteria)
+                  when Array then array_options(criteria)
+                  else
+                    raise ArgumentError,
+                          "choice criteria must be an array of options or a hash of option => description"
+                  end
+        raise ArgumentError, "choice needs at least two options" if options.size < 2
+
+        options
+      end
+
+      def array_options(list)
+        names = list.map(&:to_s)
+        duplicates = names.select { |name| names.count(name) > 1 }.uniq
+        raise ArgumentError, "choice options #{duplicates.inspect} appear twice" if duplicates.any?
+
+        names.to_h { |name| [name.dup.freeze, name.tr("_", " ").freeze] }.freeze
+      end
+    end
+  end
+end

data/lib/judge/question/noul.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Judge
+  class Question
+    class Noul < Question
+      TYPE = "noul"
+      KEYS = %w[true false].freeze
+
+      private
+
+      def normalize(criteria)
+        return nil if criteria.nil?
+
+        unless criteria.is_a?(Hash) && criteria.keys.map(&:to_s).sort == KEYS.sort
+          raise ArgumentError, "noul criteria must be a hash with exactly 'true' and 'false' keys"
+        end
+
+        entries(criteria)
+      end
+    end
+  end
+end

data/lib/judge/question/score.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Judge
+  class Question
+    class Score < Question
+      TYPE = "score"
+
+      def levels
+        criteria
+      end
+
+      def max_level
+        criteria.size - 1
+      end
+
+      private
+
+      def normalize(criteria)
+        levels = criteria.is_a?(Range) ? criteria.to_a : criteria
+        unless levels.is_a?(Array)
+          raise ArgumentError,
+                "score criteria must be an array or range of ordered levels"
+        end
+        raise ArgumentError, "score needs at least two levels" if levels.size < 2
+        unless levels.all? { |level| level.is_a?(String) || level.is_a?(Symbol) || level.is_a?(Numeric) }
+          raise ArgumentError, "score levels must be strings, symbols or numbers"
+        end
+
+        labels = levels.map { |level| level.to_s.dup.freeze }
+        raise ArgumentError, "score levels must be distinct" if labels.uniq.size != labels.size
+
+        labels.freeze
+      end
+    end
+  end
+end

data/lib/judge/question.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+
+module Judge
+  class Question
+    attr_reader :name, :instructions, :criteria, :digest
+
+    def self.type
+      self::TYPE
+    end
+
+    def initialize(instructions, criteria = nil, name: nil)
+      raise ArgumentError, "instructions must not be empty" if instructions.to_s.strip.empty?
+
+      @instructions = instructions.to_s.dup.freeze
+      @criteria = normalize(criteria).freeze
+      @name = name&.to_sym
+      @digest = Digest::SHA256.hexdigest(JSON.generate([type, @instructions, @criteria]))[0, 16].freeze
+      freeze
+    end
+
+    def type
+      self.class.type
+    end
+
+    def with_name(new_name)
+      return self if @name == new_name&.to_sym
+
+      self.class.new(@instructions, @criteria, name: new_name)
+    end
+
+    def to_payload
+      payload = { "type" => type, "instructions" => @instructions }
+      payload["criteria"] = @criteria unless @criteria.nil?
+      payload
+    end
+
+    def coerce(answer, name: @name)
+      Result.new(name: name, question: self, payload: answer)
+    end
+
+    def ==(other)
+      other.is_a?(Question) && other.type == type && other.instructions == instructions &&
+        other.criteria == criteria && other.name == name
+    end
+    alias eql? ==
+
+    def hash
+      [self.class, @instructions, @criteria, @name].hash
+    end
+
+    def inspect
+      "#<Judge::Question::#{self.class.name.split("::").last} #{@name.inspect} #{@instructions.inspect}>"
+    end
+
+    private
+
+    def normalize(criteria)
+      criteria
+    end
+
+    def entries(hash)
+      hash.to_h { |key, value| [key.to_s, entry(value, nested: false)] }.tap do |normalized|
+        duplicate_keys!(hash, normalized)
+      end.freeze
+    end
+
+    def entry(value, nested: true)
+      case value
+      when Hash then nested_entries(value)
+      when Array then value.map { |item| entry(item) }.freeze
+      when Numeric, true, false then nested ? value : value.to_s.freeze
+      else value.to_s.dup.freeze
+      end
+    end
+
+    def nested_entries(hash)
+      normalized = hash.compact.to_h { |key, value| [key.to_s, entry(value)] }
+      duplicate_keys!(hash.compact, normalized)
+      normalized.freeze
+    end
+
+    def duplicate_keys!(original, normalized)
+      return if original.size == normalized.size
+
+      keys = original.keys.map(&:to_s)
+      duplicates = keys.select { |key| keys.count(key) > 1 }.uniq
+      raise ArgumentError, "criteria keys #{duplicates.inspect} appear twice once converted to strings"
+    end
+  end
+end

data/lib/judge/rails/attributes.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module Judge
+  module Rails
+    module Attributes
+      extend ActiveSupport::Concern
+
+      included do
+        define_model_callbacks :judge_refresh, only: :after
+      end
+
+      class_methods do
+        def judge_attributes
+          @judge_attributes ||=
+            if superclass.respond_to?(:judge_attributes)
+              superclass.judge_attributes.inherit
+            else
+              Registry.new
+            end
+        end
+
+        def judge_source(source = nil, &block)
+          @judge_default_source = block || source
+        end
+
+        def judge_default_source
+          return @judge_default_source if @judge_default_source
+
+          superclass.judge_default_source if superclass.respond_to?(:judge_default_source)
+        end
+
+        def judge_attribute(name, question, source: nil, **options)
+          resolved = source || judge_default_source
+          unless resolved
+            raise ArgumentError,
+                  "judge_attribute #{name.inspect} needs a source: or a model-level judge_source"
+          end
+
+          definition = Definition.new(name: name, question: question, source: resolved, **options)
+          judge_attributes.add(definition)
+          define_judge_methods(definition)
+          judge_install_callbacks(definition) if respond_to?(:judge_install_callbacks)
+          judge_define_scopes(definition) if respond_to?(:judge_define_scopes)
+          definition
+        end
+
+        def judge_definition(name)
+          judge_attributes.fetch(name)
+        end
+
+        private
+
+        def define_judge_methods(definition)
+          name = definition.name
+
+          define_method(:"#{name}_judge_meta") { judge_meta(name) }
+          define_method(:"#{name}_probability") { judge_meta(name)["probability"] }
+          define_method(:"#{name}_confidence") { judge_meta(name)["confidence"] }
+          define_method(:"#{name}_computed_at") { judge_computed_at(name) }
+          define_method(:"#{name}_stale?") { judge_stale?(name) }
+
+          return unless definition.type == "noul"
+
+          define_method(:"#{name}?") do |threshold = 0.5|
+            value = read_attribute(name)
+            !value.nil? && value >= threshold
+          end
+        end
+      end
+
+      def judge_meta(name)
+        Storage.sidecar(self, self.class.judge_definition(name))
+      end
+
+      def judge_computed_at(name)
+        raw = judge_meta(name)["computed_at"]
+        raw.is_a?(String) ? Time.parse(raw) : raw
+      end
+
+      def judge_stale?(name = nil)
+        Storage.stale_definitions(self, judge_definitions_for(Array(name))).any?
+      end
+
+      def judge_pending
+        Storage.stale_definitions(self, judge_definitions_for([])).map(&:name)
+      end
+
+      def judge_refresh(*names, force: false, adapter: nil, &)
+        Storage.compute(self, judge_definitions_for(names), force: force, adapter: adapter, &).map(&:name)
+      end
+
+      def judge_refresh!(*names, force: false, adapter: nil, &)
+        done = []
+        Storage.compute(self, judge_definitions_for(names), force: force, adapter: adapter, done: done, &)
+      rescue StandardError
+        judge_persist_after_failure(done) if done.any?
+        raise
+      else
+        judge_persist_judgments(done, adapter: adapter) if done.any?
+        done.map(&:name)
+      end
+
+      def judge_decide(name, above:, below: nil)
+        definition = self.class.judge_definition(name)
+        probability = if definition.type == "noul"
+                        read_attribute(definition.value_column)
+                      else
+                        judge_meta(name)["probability"]
+                      end
+        raise Judge::Error, "#{name} has not been computed yet" if probability.nil?
+
+        Judge::Decision.call(probability, above: above, below: below)
+      end
+
+      private
+
+      def judge_definitions_for(names)
+        names = names.flatten
+        return self.class.judge_attributes.to_a if names.empty?
+
+        names.map { |n| self.class.judge_definition(n) }
+      end
+
+      def judge_persist_judgments(definitions, adapter: nil, inline: true)
+        return judge_create_with_judgments(adapter, inline) if new_record?
+
+        run_callbacks(:judge_refresh) do
+          next if update_columns(judge_columns_to_store(definitions))
+
+          raise ActiveRecord::RecordNotFound.new(
+            "#{self.class} #{id.inspect} no longer exists, so its new judgments were not stored",
+            self.class.name, self.class.primary_key, id
+          )
+        end
+      end
+
+      def judge_persist_after_failure(definitions)
+        judge_persist_judgments(definitions, inline: false)
+      rescue StandardError => e
+        Judge::Rails.log_failure(self.class, e)
+      end
+
+      def judge_columns_to_store(definitions)
+        columns = definitions.flat_map { |d| [d.value_column, d.sidecar_column] }
+                             .to_h { |c| [c, read_attribute(c)] }
+        now = current_time_from_proper_timezone
+        self.class.timestamp_attributes_for_update_in_model.each { |column| columns[column.to_sym] = now }
+        columns
+      end
+
+      def judge_create_with_judgments(adapter, inline)
+        @judge_refresh_adapter = adapter
+        @judge_skip_inline = !inline
+        run_callbacks(:judge_refresh) { save! }
+      ensure
+        @judge_refresh_adapter = nil
+        @judge_skip_inline = nil
+      end
+    end
+  end
+end

data/lib/judge/rails/definition.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Judge
+  module Rails
+    class Definition
+      CALLBACK_MODES = %i[async inline disabled].freeze
+      ERROR_MODES = %i[pass fail raise].freeze
+
+      attr_reader :name, :question, :source, :model, :callbacks, :if_condition, :on_error
+
+      def initialize(name:, question:, source:, model: nil, sync: false, callbacks: nil,
+                     if_condition: nil, on_error: :pass)
+        @name = name.to_sym
+        @question = question.with_name(@name)
+        @source = validate_source!(source)
+        @model = model
+        @callbacks = normalize_callbacks(callbacks, sync)
+        @if_condition = validate_condition!(if_condition)
+        @on_error = validate!(on_error.to_sym, ERROR_MODES, "on_error")
+        validate_error_mode!
+        freeze
+      end
+
+      def value_column
+        @name
+      end
+
+      def sidecar_column
+        :"#{@name}_judge"
+      end
+
+      def type
+        @question.type
+      end
+
+      def digest
+        model = effective_model
+        return @question.digest if model == Judge::Configuration::DEFAULT_MODEL
+
+        Digest::SHA256.hexdigest("#{@question.digest}:#{model}")[0, 16]
+      end
+
+      def effective_model
+        @model || Judge.config.model
+      end
+
+      def sync?
+        @callbacks == :inline
+      end
+
+      def enqueue?
+        @callbacks == :async
+      end
+
+      def state_for(record)
+        text = case @source
+               when Symbol, String then record.public_send(@source)
+               else evaluate(@source, record)
+               end
+        Array(text).reject { |part| part.to_s.strip.empty? }.join("\n\n")
+      end
+
+      def state_digest(state)
+        Digest::SHA256.hexdigest(state.to_s)[0, 16]
+      end
+
+      def applies_to?(record)
+        return true if @if_condition.nil?
+
+        @if_condition.is_a?(Proc) ? evaluate(@if_condition, record) : record.public_send(@if_condition)
+      end
+
+      def cast(result)
+        case type
+        when "noul", "score" then result.value
+        when "choice" then result.value.to_s
+        end
+      end
+
+      def sidecar(result, state_digest:, model: nil, latency: nil)
+        {
+          "digest" => digest,
+          "state_digest" => state_digest,
+          "computed_at" => Time.now.utc.iso8601,
+          "probability" => result.probability,
+          "confidence" => result.confidence,
+          "probabilities" => result.probabilities,
+          "legend" => result.legend.empty? ? nil : result.legend,
+          "model" => model,
+          "latency" => latency
+        }.compact
+      end
+
+      def stale?(value:, sidecar:, state:)
+        return true if value.nil?
+
+        meta = sidecar || {}
+        return true if meta["digest"] != digest
+
+        meta["state_digest"] != state_digest(state)
+      end
+
+      private
+
+      def evaluate(callable, record)
+        callable.arity.zero? ? record.instance_exec(&callable) : callable.call(record)
+      end
+
+      def validate_source!(source)
+        return source if source.is_a?(Symbol) || source.is_a?(String) || source.is_a?(Proc)
+
+        raise ArgumentError, "judge_attribute #{@name.inspect} source must be a Symbol, String or Proc, " \
+                             "got #{source.class}"
+      end
+
+      def validate_condition!(condition)
+        return condition if condition.nil? || condition.is_a?(Symbol) || condition.is_a?(Proc)
+
+        raise ArgumentError, "if_condition must be a Symbol or a Proc, got #{condition.class}"
+      end
+
+      def normalize_callbacks(callbacks, sync)
+        return :inline if sync && callbacks.nil?
+        return :async if callbacks.nil?
+        if sync
+          raise ArgumentError,
+                "sync: true already means inline callbacks; drop callbacks: #{callbacks.inspect}"
+        end
+
+        mode = case callbacks
+               when false then :disabled
+               when true then :async
+               else callbacks.to_s.to_sym
+               end
+        validate!(mode, CALLBACK_MODES, "callbacks")
+      end
+
+      def validate_error_mode!
+        return if @on_error == :pass || @callbacks == :inline
+
+        raise ArgumentError,
+              "on_error: #{@on_error.inspect} only applies to a synchronous attribute. " \
+              "#{@name.inspect} is #{@callbacks.inspect}, so the record is already committed by the time " \
+              "the call runs and nothing can be blocked. Pass sync: true, or leave on_error as :pass."
+      end
+
+      def validate!(value, allowed, label)
+        return value if allowed.include?(value)
+
+        raise ArgumentError, "#{label} must be one of #{allowed.inspect}, got #{value.inspect}"
+      end
+    end
+  end
+end