judge_rails 0.0.1

data/lib/generators/judge/attribute_generator.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require "rails/generators/named_base"
+require "rails/generators/active_record"
+
+module Judge
+  module Generators
+    class AttributeGenerator < ::Rails::Generators::NamedBase
+      include ::ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Adds Judge judgment attributes to a model: a migration plus the model declarations."
+
+      argument :judge_attributes, type: :array, default: [], banner: "name:noul name:choice name:score"
+      class_option :database, type: :string, aliases: %i[--db],
+                              desc: "The database whose migrations path receives the migration"
+
+      TYPES = %w[noul choice score].freeze
+      NAME = /\A[a-z_][a-z0-9_]*\z/
+
+      QUESTION_ARGUMENTS = {
+        "noul" => "",
+        "choice" => ", %w[first_option second_option third_option]",
+        "score" => ", 1..5"
+      }.freeze
+
+      def validate_pairs
+        raise ::Rails::Generators::Error, "give at least one name:type pair" if pairs.empty?
+
+        invalid = pairs.map(&:first).grep_v(NAME)
+        unless invalid.empty?
+          raise ::Rails::Generators::Error,
+                "invalid attribute name(s) #{invalid.join(", ")}: " \
+                "use lowercase letters, digits and underscores"
+        end
+
+        unknown = pairs.reject { |_, type| TYPES.include?(type) }
+        return if unknown.empty?
+
+        raise ::Rails::Generators::Error,
+              "unknown judge type(s) #{unknown.map(&:last).uniq.join(", ")}, expected #{TYPES.join(", ")}"
+      end
+
+      def create_migration_file
+        migration_template "migration.rb.tt", File.join(db_migrate_path, "#{migration_basename}.rb")
+      end
+
+      def inject_model_declarations
+        return say_status(:skip, "#{model_path} not found", :yellow) unless model_source
+        return remove_declarations if behavior == :revoke
+
+        declarations = missing_declarations
+        return say_status(:identical, model_path, :blue) if declarations.empty?
+
+        if judge_source?
+          point = insertion_point
+          unless point
+            return say_status(:skip, "could not find class #{class_name} in #{model_path}; add:\n" \
+                                     "#{declarations.join}", :yellow)
+          end
+
+          inject_into_file model_path, declarations.join, after: /\A#{Regexp.escape(model_source[0...point])}/
+        else
+          inject_into_class model_path, class_name, "#{source_line}#{declarations.join}\n"
+        end
+      end
+
+      private
+
+      def pairs
+        @pairs ||= judge_attributes.map do |pair|
+          name, type = pair.split(":", 2)
+          [name, type.to_s]
+        end
+      end
+
+      def migration_basename
+        "add_judge_#{pairs.map(&:first).join("_")}_to_#{table_name}"
+      end
+
+      def model_path
+        File.join("app", "models", class_path, "#{file_name}.rb")
+      end
+
+      def model_source
+        return @model_source if defined?(@model_source)
+
+        full = File.join(destination_root, model_path)
+        @model_source = File.exist?(full) ? File.read(full) : nil
+      end
+
+      def judge_source?
+        model_source.to_s.match?(/^\s*judge_source\b/)
+      end
+
+      def insertion_point
+        lines = model_source.lines
+        class_index = lines.index do |line|
+          line.match?(/^\s*class\s+(?:[\w:]+::)?#{Regexp.escape(class_name.demodulize)}\b/)
+        end
+        return unless class_index
+
+        indent = lines[class_index][/\A\s*/]
+        closing = ((class_index + 1)...lines.size).find { |i| lines[i].match?(/\A#{indent}end\b/) }
+        closing && lines[0...closing].join.length
+      end
+
+      def source_line
+        columns = text_columns
+        return "  judge_source { [#{columns.join(", ")}] }\n" if columns.any?
+
+        "  judge_source { [] } # TODO: the attributes Judge should read; nothing is judged until you set it\n"
+      end
+
+      def text_columns
+        model = class_name.safe_constantize
+        return [] unless model.respond_to?(:columns)
+
+        model.columns.select { |c| c.type == :text }.map(&:name).grep_v(/_judge\z/)
+      rescue StandardError
+        []
+      end
+
+      def missing_declarations
+        pairs.reject { |name, _| declared?(name) }.map { |name, type| "  #{declaration(name, type)}\n" }
+      end
+
+      def remove_declarations
+        pairs.each do |name, type|
+          gsub_file model_path, "  #{declaration(name, type)}\n", "", force: true
+        end
+        remaining = File.read(File.join(destination_root, model_path))
+        return if remaining.match?(/^\s*judge_attribute\b/)
+
+        gsub_file model_path, source_line, "", force: true
+      end
+
+      def declared?(name)
+        model_source.to_s.match?(/^\s*judge_attribute\s+:#{Regexp.escape(name)}\b/)
+      end
+
+      def declaration(name, type)
+        prompt = %("TODO: describe the judgment for #{name}"#{QUESTION_ARGUMENTS.fetch(type)})
+        "judge_attribute :#{name}, Judge.#{type}(#{prompt})"
+      end
+    end
+  end
+end

data/lib/generators/judge/install_generator.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "rails/generators/base"
+
+module Judge
+  module Generators
+    class InstallGenerator < ::Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates config/initializers/judge.rb and explains how to supply an API key."
+
+      def create_initializer
+        template "initializer.rb.tt", "config/initializers/judge.rb"
+      end
+
+      def print_next_steps
+        return unless behavior == :invoke
+
+        say ""
+        say "Judge is installed.", :green
+        say ""
+        say "  1. Provide an API key, either as the JEV_API_KEY environment variable"
+        say "     or as the Rails credential judge.api_key (bin/rails credentials:edit)."
+        say "  2. Add judgments to a model, for example:"
+        say ""
+        say "       bin/rails generate judge:attribute Ticket urgency:noul intent:choice"
+        say ""
+        say "  3. Replace the TODO questions and check judge_source: until you do, every save"
+        say "     is judged with the placeholder wording, and each judgment is a billed call."
+        say "  4. Run bin/rails db:migrate. Async attributes (the default) need ActiveJob;"
+        say "     backfill existing rows with Ticket.judge_refresh_all."
+        say ""
+      end
+    end
+  end
+end

data/lib/generators/judge/templates/initializer.rb.tt

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+Judge.configure do |config|
+  # Never commit a key. Set JEV_API_KEY (or TYPESAFE_API_KEY) in the environment, or store it
+  # in Rails credentials under judge.api_key (bin/rails credentials:edit).
+  config.api_key ||= Rails.application.credentials.dig(:judge, :api_key)
+
+  config.logger = Rails.logger
+
+  # Defaults, uncomment to override:
+  #
+  # config.base_url     = "https://api.typesafe.ai/v1/systemone"
+  # config.model        = "jev-latest"
+  # config.timeout      = 10.0
+  # config.open_timeout = 5.0
+  # config.max_retries  = 2
+  # config.max_retry_wait = 10.0
+end

data/lib/generators/judge/templates/migration.rb.tt

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class <%= migration_class_name %> < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+<% pairs.each do |name, type| -%>
+    judge_attribute :<%= table_name %>, :<%= name %>, :<%= type %>
+<% end -%>
+  end
+end

data/lib/judge/adapter.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Judge
+  # An adapter answers `call(state:, questions:, model:) -> Judge::ResultSet`, building each
+  # result with `Judge::Result.from_values`. Nothing outside an adapter knows a wire format.
+  module Adapter
+    @lock = Mutex.new
+
+    class << self
+      def registry
+        @registry ||= { jev: -> { Client.new } }
+      end
+
+      def register(name, &build)
+        raise ArgumentError, "register needs a block returning an adapter" unless build
+
+        @lock.synchronize do
+          registry[name.to_sym] = build
+          @built = nil
+        end
+        name.to_sym
+      end
+
+      def build(name)
+        return name if name.respond_to?(:call)
+
+        factory = registry[name.to_s.to_sym]
+        raise ConfigurationError, unknown(name) unless factory
+
+        factory.call
+      end
+
+      def resolve(name)
+        return name if name.respond_to?(:call)
+
+        built = @built
+        return built.last if built&.first == name
+
+        adapter = build(name)
+        @lock.synchronize do
+          @built = [name, adapter] unless @built&.first == name
+          @built.last
+        end
+      end
+
+      def names
+        registry.keys
+      end
+
+      def reset!
+        @lock.synchronize do
+          @registry = nil
+          @built = nil
+        end
+      end
+
+      def reset_built!
+        @lock.synchronize { @built = nil }
+      end
+
+      private
+
+      def unknown(name)
+        "unknown adapter #{name.inspect}. Known: #{names.map(&:inspect).join(", ")}. " \
+          "Register one with Judge::Adapter.register(#{name.to_s.to_sym.inspect}) { ... }"
+      end
+    end
+  end
+end

data/lib/judge/client.rb

@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "openssl"
+require "zlib"
+require "uri"
+
+module Judge
+  class Client
+    BACKOFF_BASE = 0.25
+    BACKOFF_CAP = 8.0
+    CONNECTIONS_KEY = :judge_client_connections
+    TRANSPORT_ERRORS = [Timeout::Error, SocketError, IOError, SystemCallError, OpenSSL::SSL::SSLError,
+                        Net::ProtocolError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Zlib::Error].freeze
+    UNSAFE_TO_RESEND = [Net::ReadTimeout].freeze
+    RETRYABLE_STATUSES = (500..599)
+
+    attr_reader :config
+    attr_accessor :sleeper
+
+    def self.close_thread_connections
+      store = Thread.current[CONNECTIONS_KEY]
+      Thread.current[CONNECTIONS_KEY] = nil
+      return unless store && store[:pid] == Process.pid
+
+      store[:connections].each_value do |http|
+        http.finish if http.started?
+      rescue IOError
+        nil
+      end
+    end
+
+    def inspect
+      "#<Judge::Client base_url=#{@config.base_url.inspect} model=#{@config.model.inspect}>"
+    end
+
+    def initialize(config: nil, sleeper: nil)
+      @config = config || Judge.config
+      @sleeper = sleeper || Kernel.method(:sleep)
+    end
+
+    def call(state:, questions:, model: nil)
+      raise ArgumentError, "state must not be nil" if state.nil?
+      raise ArgumentError, "questions must not be empty" if questions.nil? || questions.empty?
+
+      payload = JSON.generate(build_payload(state, questions, model))
+      event = { model: model || @config.model, questions: questions.size,
+                request_bytes: payload.bytesize }
+
+      instrument(event) do
+        started = monotonic
+        body = perform(payload)
+        set = ResultSet.from_response(body, questions: questions, latency: monotonic - started)
+        event[:latency] = set.latency
+        event[:input_tokens] = set.usage&.input_tokens
+        event[:output_tokens] = set.usage&.output_tokens
+        set
+      end
+    end
+
+    private
+
+    def instrument(event, &)
+      return yield unless defined?(::ActiveSupport::Notifications)
+
+      ::ActiveSupport::Notifications.instrument("request.judge", event, &)
+    end
+
+    def build_payload(state, questions, model)
+      {
+        "state" => state,
+        "model" => model || @config.model,
+        "questions" => questions.to_h { |name, question| [name.to_s, question.to_payload] }
+      }
+    end
+
+    def perform(payload)
+      authorization = "Bearer #{@config.api_key!}"
+      attempt = 0
+
+      loop do
+        attempt += 1
+        response = attempt_request(authorization, payload, attempt)
+        next if response.nil?
+
+        status = response.code.to_i
+        return parse_body(response) if status < 300
+
+        raise error_for(status, response) unless retryable?(status) && attempt <= @config.max_retries
+
+        pause(retry_delay(response, status, attempt))
+      end
+    end
+
+    def retry_delay(response, status, attempt)
+      after = [429, 503].include?(status) ? retry_after(response) : nil
+      raise error_for(status, response) if after && @config.max_retry_wait && after > @config.max_retry_wait
+
+      after || backoff(attempt)
+    end
+
+    def attempt_request(authorization, payload, attempt)
+      started = monotonic
+      response = execute(authorization, payload)
+      log(response.code, monotonic - started, attempt)
+      response
+    rescue *TRANSPORT_ERRORS => e
+      close_connection
+      log(e.class.name, monotonic - started, attempt)
+      if attempt > @config.max_retries || UNSAFE_TO_RESEND.any? { |klass| e.is_a?(klass) }
+        raise TransportError, "#{e.class}: #{e.message}"
+      end
+
+      pause(backoff(attempt))
+      nil
+    end
+
+    def execute(authorization, payload)
+      request = Net::HTTP::Post.new(uri.request_uri)
+      request["Authorization"] = authorization
+      request["Content-Type"] = "application/json"
+      request["Accept"] = "application/json"
+      request.body = payload
+      connection.request(request)
+    end
+
+    def uri
+      url = @config.base_url
+      @uri = URI.parse(url) if @uri_source != url
+      @uri_source = url
+      @uri
+    end
+
+    def connection_key
+      [@config.base_url, @config.open_timeout, @config.timeout].freeze
+    end
+
+    def connection
+      store = connections
+      key = connection_key
+      http = store[key]
+      return http if http&.started?
+
+      store[key] = start_connection(key.first)
+    end
+
+    def start_connection(base_url)
+      uri = URI.parse(base_url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == "https"
+      http.open_timeout = @config.open_timeout
+      http.read_timeout = @config.timeout
+      http.write_timeout = @config.timeout if http.respond_to?(:write_timeout=)
+      http.keep_alive_timeout = 30
+      http.start
+      http
+    end
+
+    def connections
+      store = Thread.current[CONNECTIONS_KEY]
+      unless store && store[:pid] == Process.pid
+        store = { pid: Process.pid, connections: {} }
+        Thread.current[CONNECTIONS_KEY] = store
+      end
+      store[:connections]
+    end
+
+    def close_connection
+      http = connections.delete(connection_key)
+      http.finish if http&.started?
+    rescue IOError
+      nil
+    end
+
+    def parse_body(response)
+      body = response.body.to_s
+      parsed = JSON.parse(body)
+      raise InvalidResponseError, "expected a JSON object, got #{parsed.class}" unless parsed.is_a?(Hash)
+
+      parsed
+    rescue JSON::ParserError => e
+      raise InvalidResponseError, "could not parse response body: #{e.message}"
+    end
+
+    def retryable?(status)
+      status == 429 || RETRYABLE_STATUSES.cover?(status)
+    end
+
+    def retry_after(response)
+      value = response["Retry-After"]
+      return nil if value.nil?
+
+      Float(value)
+    rescue ArgumentError, TypeError
+      nil
+    end
+
+    def backoff(attempt)
+      window = [BACKOFF_BASE * (2**(attempt - 1)), BACKOFF_CAP].min
+      window * (0.5 + (rand * 0.5))
+    end
+
+    def pause(seconds)
+      @sleeper.call(seconds) if seconds.positive?
+    end
+
+    def error_for(status, response)
+      body = response.body.to_s
+      message = "Judge API returned #{status}: #{error_message(body)}"
+
+      case status
+      when 401, 403 then AuthenticationError.new(message, status: status, body: body)
+      when 400, 404, 422 then InvalidRequestError.new(message, status: status, body: body)
+      when 413 then PayloadTooLargeError.new(message, status: status, body: body)
+      when 429
+        RateLimitError.new(message, status: status, body: body, retry_after: retry_after(response))
+      when RETRYABLE_STATUSES then ServerError.new(message, status: status, body: body)
+      else APIError.new(message, status: status, body: body)
+      end
+    end
+
+    def error_message(body)
+      parsed = JSON.parse(body)
+      return truncate(body) unless parsed.is_a?(Hash)
+
+      error = parsed["error"]
+      message = error.is_a?(Hash) ? error["message"] : error
+      (message || parsed["message"] || truncate(body)).to_s
+    rescue JSON::ParserError
+      truncate(body)
+    end
+
+    def truncate(body, limit = 200)
+      body.length > limit ? "#{body[0, limit]}..." : body
+    end
+
+    def log(status, latency, attempt)
+      logger = @config.logger
+      return unless logger
+
+      logger.debug do
+        "Judge POST #{uri.path} status=#{status} latency=#{latency.round(3)}s attempt=#{attempt}"
+      end
+    end
+
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end
+
+Judge::Pool.on_worker_exit { Judge::Client.close_thread_connections }

data/lib/judge/configuration.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Judge
+  class Configuration
+    DEFAULT_BASE_URL = "https://api.typesafe.ai/v1/systemone"
+    DEFAULT_MODEL = "jev-latest"
+
+    attr_reader :api_key, :base_url, :model, :timeout, :open_timeout, :max_retries, :max_retry_wait,
+                :concurrency, :adapter
+    attr_accessor :logger
+
+    def initialize
+      self.api_key = [ENV.fetch("JEV_API_KEY", nil), ENV.fetch("TYPESAFE_API_KEY", nil)]
+                     .find { |value| !blank?(value) }
+      self.base_url = present_env("JEV_BASE_URL") || DEFAULT_BASE_URL
+      self.model = present_env("JEV_MODEL") || DEFAULT_MODEL
+      self.timeout = 10.0
+      self.open_timeout = 5.0
+      self.max_retries = 2
+      self.max_retry_wait = 10.0
+      self.concurrency = nil
+      self.adapter = present_env("JUDGE_ADAPTER") || :jev
+      @logger = nil
+    end
+
+    def api_key=(value)
+      @api_key = blank?(value) ? nil : value.to_s.strip
+    end
+
+    def base_url=(value)
+      raise ArgumentError, "base_url must not be blank" if blank?(value)
+
+      @base_url = value.to_s.strip
+    end
+
+    def model=(value)
+      raise ArgumentError, "model must not be blank" if blank?(value)
+
+      @model = value.to_s.strip
+    end
+
+    def timeout=(value)
+      @timeout = positive_number(value, "timeout")
+    end
+
+    def open_timeout=(value)
+      @open_timeout = positive_number(value, "open_timeout")
+    end
+
+    def max_retries=(value)
+      unless value.is_a?(Integer) && !value.negative?
+        raise ArgumentError, "max_retries must be an Integer of 0 or more, got #{value.inspect}"
+      end
+
+      @max_retries = value
+    end
+
+    def max_retry_wait=(value)
+      @max_retry_wait = value.nil? ? nil : positive_number(value, "max_retry_wait")
+    end
+
+    def concurrency=(value)
+      unless value.nil? || (value.is_a?(Integer) && value.positive?)
+        raise ArgumentError, "concurrency must be nil or a positive Integer, got #{value.inspect}"
+      end
+
+      @concurrency = value
+    end
+
+    def adapter=(value)
+      @adapter = value.respond_to?(:call) ? value : value.to_s.strip.to_sym
+    end
+
+    def api_key!
+      return @api_key if @api_key
+
+      raise ConfigurationError, "No Judge API key. Set JEV_API_KEY or Judge.configure { |c| c.api_key = ... }"
+    end
+
+    def inspect
+      key = @api_key ? "[FILTERED]" : "nil"
+      "#<Judge::Configuration api_key=#{key} base_url=#{@base_url.inspect} model=#{@model.inspect} " \
+        "adapter=#{@adapter.inspect}>"
+    end
+
+    private
+
+    def present_env(name)
+      value = ENV.fetch(name, nil)
+      blank?(value) ? nil : value.strip
+    end
+
+    def blank?(value)
+      value.nil? || value.to_s.strip.empty?
+    end
+
+    def positive_number(value, label)
+      return value.to_f if value.is_a?(Numeric) && value.positive?
+
+      raise ArgumentError, "#{label} must be a positive number of seconds, got #{value.inspect}"
+    end
+  end
+end

data/lib/judge/decision.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Judge
+  module Decision
+    module_function
+
+    def call(probability, above:, below: nil)
+      low = band_floor(above, below)
+      return :yes if probability >= above
+      return :no if probability <= low
+
+      :unsure
+    end
+
+    def band_floor(above, below)
+      low = below || [(1.0 - above).round(12), above].min
+      return low if low <= above
+
+      raise ArgumentError,
+            "decide needs below (#{low}) to be at or under above (#{above}), " \
+            "otherwise no probability can land in :no or :unsure"
+    end
+  end
+end

data/lib/judge/errors.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Judge
+  class Error < StandardError; end
+
+  class ConfigurationError < Error; end
+
+  class TransportError < Error; end
+
+  class APIError < Error
+    attr_reader :status, :body
+
+    def initialize(message, status: nil, body: nil)
+      @status = status
+      @body = body
+      super(message)
+    end
+  end
+
+  class AuthenticationError < APIError; end
+  class InvalidRequestError < APIError; end
+  class ServerError < APIError; end
+
+  class RateLimitError < APIError
+    attr_reader :retry_after
+
+    def initialize(message, status: nil, body: nil, retry_after: nil)
+      @retry_after = retry_after
+      super(message, status: status, body: body)
+    end
+  end
+
+  class PayloadTooLargeError < APIError; end
+
+  class InvalidResponseError < Error; end
+end