judgelint 0.1.0-x86_64-linux

data/lib/judgelint/client.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+module Judgelint
+  # Asks one model (an Endpoint) the questions over Net::HTTP, in whichever
+  # protocol its provider speaks. Plain Net::HTTP rather than a gem keeps the
+  # same code compiling with Spinel.
+  class Client
+    RETRY_STATUSES = %w[408 429 500 502 503 504 529].freeze
+    # The native binary links OpenSSL statically, so it looks for certificates
+    # where OpenSSL was built. Unless SSL_CERT_FILE says otherwise, it uses the
+    # system's bundle: macOS and Alpine, Debian and Ubuntu, then Fedora and RHEL.
+    CA_BUNDLES = %w[/etc/ssl/cert.pem /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt].freeze
+
+    attr_reader :usage, :endpoint
+
+    def initialize(endpoint, usage: Usage.new, sleeper: ->(seconds) { sleep(seconds) })
+      if RUBY_ENGINE == "spinel" && ENV.fetch("SSL_CERT_FILE", "").empty?
+        bundle = CA_BUNDLES.find { |path| File.file?(path) }
+        ENV["SSL_CERT_FILE"] = bundle if bundle
+      end
+      @endpoint = endpoint
+      @protocol = Protocol::ALL.fetch(endpoint.protocol) do
+        raise ConfigError, "protocol is system_one or decisions, not #{endpoint.protocol}"
+      end
+      @usage = usage
+      @sleeper = sleeper
+      check_settings
+    end
+
+    # Returns the probability of "yes" for each question, keyed by its key,
+    # or nil for a question the model refused.
+    def judge(state, questions)
+      data = self.class.unwrap(post(JSON.generate(payload(state, questions))))
+      @usage.record(data, @endpoint.price_per_million)
+      @protocol.parse(data, questions)
+    end
+
+    def payload(state, questions)
+      @protocol.payload(@endpoint.model, state, questions).merge(@endpoint.provider_options)
+    end
+
+    # A conservative token count for code: three bytes a token.
+    def tokens(state, questions)
+      JSON.generate(payload(state, questions)).bytesize / 3
+    end
+
+    # What one more question adds to a request, without serializing the file again.
+    def question_tokens(question)
+      (JSON.generate(@protocol.render(question)).bytesize + question.key.bytesize + 6) / 3
+    end
+
+    def url
+      base = Config.expand(@endpoint.api_base)
+      path = Config.expand(@endpoint.path).gsub("{model}", @endpoint.model.to_s)
+      "#{base.delete_suffix('/')}/#{path.delete_prefix('/')}"
+    end
+
+    # Cloudflare wraps the answers as {result: {...}, success, errors}.
+    def self.unwrap(data)
+      raise ApiError, "Response is not a JSON object" unless data.is_a?(Hash)
+
+      result = data["result"]
+      !data.key?("answers") && result.is_a?(Hash) && result.key?("answers") ? result : data
+    end
+
+    # The error message, whichever of the shapes providers use: TypeSafe's
+    # detail (a string, {message}, or a validation list), OpenAI's
+    # {error: {message}}, Cloudflare's errors list, or a bare message.
+    def self.error_detail(body)
+      data = JSON.parse(body)
+      return body[0, 200] unless data.is_a?(Hash)
+
+      detail = data["detail"] || data["error"] || data["errors"] || data["message"]
+      message(detail) || body[0, 200]
+    rescue JSON::ParserError
+      body[0, 200]
+    end
+
+    def self.message(detail)
+      case detail
+      when String then detail.empty? ? nil : detail
+      when Hash then message(detail["message"] || detail["msg"])
+      when Array
+        messages = detail.map { |item| message(item) }.compact
+        messages.empty? ? nil : messages.join("; ")
+      end
+    end
+
+    private
+
+    def check_settings
+      provider = @endpoint.provider
+      if @endpoint.api_base.to_s.empty?
+        raise ConfigError, "Provider #{provider} needs #{@endpoint.needs || 'api_base'} in .judgelint.yml"
+      end
+      raise ConfigError, "Provider #{provider} needs a model in .judgelint.yml" if model_needed?
+      raise ConfigError, "Set #{@endpoint.api_key_env} to your #{provider} API key" if key_missing?
+
+      unless %w[bearer basic none].include?(@endpoint.auth) || @endpoint.auth.start_with?("header:")
+        raise ConfigError, "auth is bearer, basic, none, or header:<name>, not #{@endpoint.auth}"
+      end
+    end
+
+    # llama.cpp serves one model and doesn't need its name; nothing else can guess it.
+    def model_needed?
+      @endpoint.model.to_s.empty? && @endpoint.provider != "llamacpp"
+    end
+
+    def key_missing?
+      @endpoint.auth != "none" && !@endpoint.api_key_env.to_s.empty? && @endpoint.api_key.to_s.empty?
+    end
+
+    def post(body)
+      attempt = 0
+      loop do
+        attempt += 1
+        response = request(body, attempt)
+        next if response.nil?
+
+        code = response.code
+        return parse_json(response.body.to_s) if code.start_with?("2")
+
+        detail = self.class.error_detail(response.body.to_s)
+        if RETRY_STATUSES.include?(code) && attempt <= Endpoint::RETRIES
+          @sleeper.call(retry_delay(response, attempt))
+          next
+        end
+        raise ApiError, "#{@endpoint.label} answered #{code}: #{detail}"
+      end
+    end
+
+    # Answers nil after a connection error that's worth another try.
+    def request(body, attempt)
+      uri = URI(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == "https"
+      http.open_timeout = Endpoint::TIMEOUT
+      http.read_timeout = Endpoint::TIMEOUT
+
+      request = Net::HTTP::Post.new(uri.request_uri)
+      request["Content-Type"] = "application/json"
+      request["Accept"] = "application/json"
+      authorize(request)
+      @endpoint.headers.each { |name, value| request[name.to_s] = Config.expand(value.to_s) }
+      request.body = body
+      http.request(request)
+    rescue Judgelint::Error
+      raise
+    rescue StandardError => e
+      # Nothing listening isn't going to change in a few seconds; a dropped or
+      # stalled connection might.
+      refused = e.is_a?(Errno::ECONNREFUSED) || e.message.include?("Connection refused")
+      raise ApiError, "Couldn't reach #{@endpoint.label} at #{url}: #{e.message}" if refused || attempt > Endpoint::RETRIES
+
+      @sleeper.call(2.0**attempt)
+      nil
+    end
+
+    def authorize(request)
+      key = @endpoint.api_key
+      return if key.to_s.empty?
+
+      case @endpoint.auth
+      when "bearer" then request["Authorization"] = "Bearer #{key}"
+      when "basic" then request["Authorization"] = "Basic #{["token:#{key}"].pack("m0")}"
+      when "none" then nil
+      else request[@endpoint.auth.delete_prefix("header:")] = key
+      end
+    end
+
+    def retry_delay(response, attempt)
+      after = Judgelint.number(response["Retry-After"])
+      after ? [after, 30.0].min : 0.5 * (2**(attempt - 1))
+    end
+
+    def parse_json(body)
+      JSON.parse(body)
+    rescue JSON::ParserError
+      raise ApiError, "#{@endpoint.label} answered with something other than JSON: #{body[0, 200]}"
+    end
+  end
+end

data/lib/judgelint/config.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module Judgelint
+  # .judgelint.yml: which models to ask, the threshold, files to ignore, and a
+  # spending limit. Everything else has a fixed value.
+  #
+  #   models: [typesafe, cloudflare:clef]   # ask both and average
+  #   threshold: 0.7
+  #   ignore: ["vendor/**"]
+  #   max_cost_usd: 1.00
+  class Config
+    FILE = ".judgelint.yml"
+    KEYS = %w[models threshold ignore max_cost_usd].freeze
+    RULES_DIR = ".judgelint/rules"
+    CACHE = ".judgelint/cache.json"
+    WINDOW_LINES = 40
+    THREADS = 8
+    # Answers this far below the threshold, or less, are uncertain: reported, never failing.
+    UNCERTAIN_BAND = 0.3
+
+    attr_reader :models, :threshold, :ignore, :max_cost_usd
+
+    # +overrides+ replaces the file's settings: --model replaces its models.
+    def self.load(path: nil, overrides: {})
+      new(Judgelint.read_yaml(path || FILE, required: !path.nil?).merge(overrides))
+    end
+
+    # ${VAR} reads the environment, so a URL can carry an account id. The
+    # client expands it when it sends, so commands that make no request don't
+    # need every variable set.
+    def self.expand(value)
+      value.gsub(/\$\{[A-Za-z_][A-Za-z0-9_]*\}/) do |reference|
+        name = reference[2..-2]
+        found = ENV.fetch(name, "")
+        raise ConfigError, "Set #{name} (used in #{value})" if found.empty?
+
+        found
+      end
+    end
+
+    def initialize(data)
+      unknown = data.keys - KEYS
+      raise ConfigError, "#{FILE}: unknown keys #{unknown.join(', ')} (it takes #{KEYS.join(', ')})" unless unknown.empty?
+
+      entries = data.fetch("models", "typesafe")
+      entries = [entries] unless entries.is_a?(Array)
+      raise ConfigError, "models needs at least one model" if entries.empty?
+
+      @models = entries.map { |entry| Endpoint.from(entry) }
+      @threshold = Judgelint.number(data.key?("threshold") ? data["threshold"] : 0.7)
+      raise ConfigError, "threshold must be a number between 0 and 1" unless @threshold&.between?(0.01, 1)
+
+      @ignore = list(data["ignore"])
+      @max_cost_usd = Judgelint.number(data.key?("max_cost_usd") ? data["max_cost_usd"] : 1)
+      raise ConfigError, "max_cost_usd must be a number" if @max_cost_usd.nil?
+    end
+
+    def warn_at
+      [threshold - UNCERTAIN_BAND, 0.0].max
+    end
+
+    def label
+      models.map(&:label).join(" + ")
+    end
+
+    private
+
+    def list(value)
+      return [] if value.nil?
+
+      value.is_a?(Array) ? value.map(&:to_s) : [value.to_s]
+    end
+  end
+
+  # One model to ask, and how to reach it: a provider preset and model id
+  # ("typesafe", "cloudflare:clef", "openrouter:jaredpalmer/kev-4b"), or a
+  # mapping for an endpoint judgelint has no preset for.
+  class Endpoint
+    KEYS = %w[
+      provider model api_base path protocol auth api_key_env headers provider_options
+      price_per_million batch_size max_state_tokens max_request_tokens
+    ].freeze
+    TIMEOUT = 30
+    RETRIES = 3
+
+    attr_reader :provider, :model, :api_base, :path, :protocol, :auth, :api_key_env, :headers,
+                :provider_options, :price_per_million, :batch_size, :max_state_tokens, :max_request_tokens, :needs
+
+    def self.from(entry)
+      return new(entry) if entry.is_a?(Hash)
+
+      provider, model = entry.to_s.split(":", 2)
+      data = { "provider" => provider }
+      data["model"] = model if model && !model.empty?
+      new(data)
+    end
+
+    def initialize(data)
+      unknown = data.keys - KEYS
+      raise ConfigError, "models: unknown keys #{unknown.join(', ')}" unless unknown.empty?
+
+      @data = data
+      @provider = data.fetch("provider", "custom").to_s
+      @preset = PROVIDERS.fetch(@provider, {})
+      @model = setting("model")
+      @api_base = setting("api_base")
+      @path = setting("path") || "/v1/systemone"
+      @protocol = setting("protocol") || "system_one"
+      @auth = setting("auth") || "bearer"
+      @api_key_env = setting("api_key_env")
+      @headers = mapping("headers")
+      @provider_options = mapping("provider_options")
+      @price_per_million = price
+      @batch_size = count("batch_size", 32)
+      # A request's state plus its widest question, and a whole request: Jev's
+      # 32k and 64k token limits, with room for the estimate to be off.
+      @max_state_tokens = count("max_state_tokens", 30_000)
+      @max_request_tokens = count("max_request_tokens", 60_000)
+      @needs = @preset["needs"]
+    end
+
+    def label
+      "#{provider}:#{model}"
+    end
+
+    def api_key
+      api_key_env.to_s.empty? ? nil : ENV.fetch(api_key_env, nil)
+    end
+
+    private
+
+    def setting(key)
+      value = @data.key?(key) ? @data[key] : @preset[key]
+      value.nil? ? nil : value.to_s
+    end
+
+    def mapping(key)
+      value = @data.fetch(key, {})
+      raise ConfigError, "models: #{key} must be a mapping" unless value.is_a?(Hash)
+
+      value
+    end
+
+    # A preset's prices are per model; another model has no price unless the
+    # entry gives one, and its cost shows as unknown.
+    def price
+      return Judgelint.number(@data["price_per_million"]) if @data.key?("price_per_million")
+
+      prices = @preset["prices"] || {}
+      prices.key?(@model.to_s) ? Judgelint.number(prices[@model.to_s]) : nil
+    end
+
+    def count(key, default)
+      # Not fetch(key, default): Spinel miscompiles a Float default on a YAML hash.
+      value = @data.key?(key) ? @data[key] : @preset[key]
+      number = Judgelint.number(value.nil? ? default : value)
+      raise ConfigError, "models: #{key} must be a whole number of at least 1" if number.nil? || number < 1
+
+      number.to_i
+    end
+  end
+end

data/lib/judgelint/errors.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Judgelint
+  class Error < StandardError; end
+
+  # The command line doesn't parse.
+  class UsageError < Error; end
+
+  # A rule file or the config file can't be used as written.
+  class ConfigError < Error; end
+
+  # The decision model couldn't be reached or answered with something unusable.
+  class ApiError < Error; end
+end

data/lib/judgelint/glob.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Judgelint
+  # Path matching shared by rule targeting and file discovery. `**/` matches
+  # zero or more directories and `{a,b}` alternates, as in Dir.glob, and a
+  # trailing `/**` matches everything under a directory.
+  module Glob
+    FLAGS = File::FNM_PATHNAME | File::FNM_EXTGLOB
+
+    def self.match_any?(patterns, path)
+      patterns.any? do |pattern|
+        File.fnmatch?(pattern, path, FLAGS) || (pattern.end_with?("/**") && File.fnmatch?("#{pattern}/*", path, FLAGS))
+      end
+    end
+
+    def self.expand(patterns)
+      patterns.flat_map { |pattern| Dir.glob(pattern) }.select { |path| File.file?(path) }.uniq.sort
+    end
+
+    # Files named on the command line, with directories expanded.
+    def self.expand_arguments(arguments)
+      arguments.flat_map do |argument|
+        if File.directory?(argument)
+          expand([File.join(argument, "**", "*")])
+        else
+          [argument]
+        end
+      end.map { |path| path.delete_prefix("./") }.uniq
+    end
+  end
+end

data/lib/judgelint/pool.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Judgelint
+  # Maps items over a few threads and returns results in the items' order, so
+  # output doesn't depend on which request came back first. The first error
+  # is raised once every thread has finished.
+  module Pool
+    def self.map(items, size)
+      results = Array.new(items.size)
+      queue = Queue.new
+      items.each_with_index { |item, index| queue << [item, index] }
+      errors = []
+      lock = Mutex.new
+      threads = Array.new([[size, items.size].min, 1].max) do
+        Thread.new do
+          Thread.current.report_on_exception = false
+          loop do
+            pair = next_item(queue)
+            break if pair.nil?
+
+            begin
+              results[pair[1]] = yield(pair[0])
+            rescue StandardError => e
+              lock.synchronize { errors << e }
+            end
+          end
+        end
+      end
+      threads.each(&:join)
+      raise errors.first unless errors.empty?
+
+      results
+    end
+
+    def self.next_item(queue)
+      queue.pop(true)
+    rescue ThreadError
+      nil
+    end
+  end
+end

data/lib/judgelint/protocol.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Judgelint
+  # The two request shapes decision models are served with. judgelint asks only
+  # yes/no questions, so each renders a Question as that protocol's yes/no
+  # type and reads back the probability of yes. A nil probability is a refusal.
+  module Protocol
+    # TypeSafe's System One: questions keyed by name, yes/no is "noul".
+    module SystemOne
+      def self.payload(model, state, questions)
+        rendered = {}
+        questions.each { |question| rendered[question.key] = render(question) }
+        body = { "state" => state, "questions" => rendered }
+        body["model"] = model unless model.to_s.empty?
+        body
+      end
+
+      def self.render(question)
+        rendered = { "type" => "noul", "instructions" => question.instructions }
+        criteria = {}
+        criteria["true"] = question.yes_means if question.yes_means
+        criteria["false"] = question.no_means if question.no_means
+        rendered["criteria"] = criteria unless criteria.empty?
+        rendered
+      end
+
+      def self.parse(data, questions)
+        answers = data["answers"]
+        raise ApiError, "Response has no answers: #{JSON.generate(data)[0, 200]}" unless answers.is_a?(Hash)
+
+        probabilities = {}
+        questions.each do |question|
+          probabilities[question.key] = Protocol.probability(answers[question.key], "noul", question)
+        end
+        probabilities
+      end
+    end
+
+    # OpenAI's Decisions API: questions are a list with names, yes/no is
+    # "predicate" with no criteria field, and answers come back as a list.
+    module Decisions
+      # Decisions' input is text, so a structured state goes as its JSON.
+      def self.payload(model, state, questions)
+        input = state.is_a?(String) ? state : JSON.generate(state)
+        { "model" => model, "input" => input, "questions" => questions.map { |question| render(question) } }
+      end
+
+      # Criteria become lines of the instructions, as RubyLLM and LiteLLM do.
+      def self.render(question)
+        lines = [question.instructions]
+        lines << "Yes: #{question.yes_means}" if question.yes_means
+        lines << "No: #{question.no_means}" if question.no_means
+        { "type" => "predicate", "name" => question.key, "instructions" => lines.join("\n") }
+      end
+
+      def self.parse(data, questions)
+        answers = data["answers"]
+        raise ApiError, "Response has no answers: #{JSON.generate(data)[0, 200]}" unless answers.is_a?(Array)
+
+        # Answers come back in question order; match by name when there is
+        # one, and give unnamed answers to the remaining questions in order.
+        by_name = {}
+        unnamed = []
+        answers.each do |answer|
+          name = answer.is_a?(Hash) ? answer["name"].to_s : ""
+          name.empty? ? unnamed << answer : by_name[name] = answer
+        end
+        probabilities = {}
+        questions.each do |question|
+          answer = by_name.key?(question.key) ? by_name[question.key] : unnamed.shift
+          probabilities[question.key] = Protocol.probability(answer, "probability", question)
+        end
+        probabilities
+      end
+    end
+
+    ALL = { "system_one" => SystemOne, "decisions" => Decisions }.freeze
+
+    def self.probability(answer, field, question)
+      return nil if answer.is_a?(Hash) && answer["type"] == "refusal"
+
+      value = answer.is_a?(Hash) ? answer[field] : nil
+      raise ApiError, "No yes/no answer for #{question.key}: #{answer.inspect}" unless value.is_a?(Numeric)
+
+      value.to_f
+    end
+  end
+end

data/lib/judgelint/providers.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Judgelint
+  # Presets for the services and servers that answer System One questions.
+  # Any key can be overridden in .judgelint.yml, and a provider not listed here
+  # works by setting api_base (and model, path, protocol, auth as needed).
+  #
+  # protocol is "system_one" (POST {model, state, questions} keyed by name) or
+  # "decisions" (OpenAI's Decisions API: input, questions as a list). The path
+  # doesn't decide it: Perplexity and Hanzo take System One at /v1/decisions.
+  #
+  # auth is "bearer", "basic" (user "token", for Databricks), "none", or
+  # "header:<name>" (Microsoft Foundry's api-key). A preset with no
+  # api_key_env sends no credentials unless you set one.
+  #
+  # ${VAR} in api_base or path reads the environment; {model} in path is the
+  # model id. batch_size is the provider's most questions per request.
+  # prices are USD per million input tokens, by model (OpenRouter reports
+  # each request's cost instead). needs says what a preset can't fill in.
+  PROVIDERS = {
+    "typesafe" => {
+      "api_base" => "https://api.typesafe.ai", "api_key_env" => "TYPESAFE_API_KEY", "model" => "jev-latest",
+      "prices" => { "jev-latest" => "0.042", "jev-preview" => "0.042" }
+    },
+    "openrouter" => {
+      "api_base" => "https://openrouter.ai/api", "api_key_env" => "OPENROUTER_API_KEY", "model" => "typesafe/jev-1.13"
+    },
+    "celeris" => {
+      "api_base" => "https://inference.celeris.ai/celeris-1-decision", "api_key_env" => "CELERIS_API_KEY",
+      "model" => "celeris-1-decision", "prices" => { "celeris-1-decision" => "0.04" }
+    },
+    "cloudflare" => {
+      "api_base" => "https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/ai/run",
+      "path" => "/@cf/cloudflare/{model}", "api_key_env" => "CLOUDFLARE_API_TOKEN", "model" => "clef",
+      "batch_size" => "64", "prices" => { "clef" => "0.24" }
+    },
+    "openai" => {
+      "api_base" => "https://api.openai.com", "path" => "/v1/decisions", "protocol" => "decisions",
+      "api_key_env" => "OPENAI_API_KEY", "model" => "gpt-6-luna", "prices" => { "gpt-6-luna" => "0.10" }
+    },
+    "perplexity" => {
+      "api_base" => "https://api.perplexity.ai", "path" => "/v1/decisions", "api_key_env" => "PERPLEXITY_API_KEY",
+      "model" => "pplx-decider-latest", "batch_size" => "128",
+      "prices" => { "pplx-decider-latest" => "0.02", "pplx-decider-flash-latest" => "0.01" }
+    },
+    "vercel" => {
+      "api_base" => "https://ai-gateway.vercel.sh/typesafe", "api_key_env" => "AI_GATEWAY_API_KEY",
+      "model" => "typesafe-ai/jev"
+    },
+    "digitalocean" => {
+      "api_base" => "https://inference.do-ai.run", "api_key_env" => "MODEL_ACCESS_KEY", "model" => "typesafe-jev-1.13.0"
+    },
+    "upstage" => {
+      "api_base" => "https://api.upstage.ai", "api_key_env" => "UPSTAGE_API_KEY", "model" => "solar-decide"
+    },
+    "hanzo" => {
+      "api_base" => "https://api.hanzo.ai", "path" => "/v1/decisions", "api_key_env" => "HANZO_API_KEY",
+      "model" => "kai", "batch_size" => "100", "prices" => { "kai" => "0.021" }
+    },
+    "foundry" => {
+      "path" => "/providers/microsoft/v1/systemone", "auth" => "header:api-key", "api_key_env" => "AZURE_API_KEY",
+      "needs" => "api_base (your Foundry resource endpoint) and model (your deployment name)"
+    },
+    "databricks" => {
+      "path" => "/ai-gateway/typesafe/v1/systemone", "auth" => "basic", "api_key_env" => "DATABRICKS_TOKEN",
+      "needs" => "api_base (https://<workspace host>) and model (a Unity Catalog name)"
+    },
+    "ollama" => {
+      "api_base" => "http://localhost:11434", "batch_size" => "64", "needs" => "model (a decision model you've pulled)"
+    },
+    "llamacpp" => { "api_base" => "http://localhost:8080" },
+    "vllm" => { "api_base" => "http://localhost:8000", "batch_size" => "64", "needs" => "model (the model the server runs)" },
+    "litellm" => {
+      "api_base" => "http://localhost:4000", "api_key_env" => "LITELLM_API_KEY",
+      "needs" => "model (a decision model routed by your proxy)"
+    }
+  }.freeze
+end