judgelint 0.1.0

data/lib/judgelint/question.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Judgelint
+  # One rule asked about one range of a file. The model sees the whole file;
+  # the question points at the lines.
+  #
+  # A spec's bullet is asked twice: whether the code breaks it, and, with the
+  # answer flipped, whether the code follows it. Decision models lean toward
+  # no on the first and yes on the second, so the average is steadier than
+  # either.
+  class Question
+    BREAKS = "The code clearly breaks the rule."
+    FOLLOWS = "The code follows the rule, or the rule doesn't apply to it."
+
+    attr_reader :rule, :first, :last
+
+    def initialize(rule, first, last, inverted: false)
+      @rule = rule
+      @first = first
+      @last = last
+      @inverted = inverted
+    end
+
+    def key
+      base = "#{rule.question_key}.#{first}-#{last}"
+      return base unless rule.convention
+
+      @inverted ? "#{base}.follows" : "#{base}.breaks"
+    end
+
+    # The probability the code breaks the rule, from the probability of yes.
+    def violation(yes_probability)
+      @inverted ? 1.0 - yes_probability : rule.violation_probability(yes_probability)
+    end
+
+    # "Answer yes only when..." keeps the model from flagging code the rule
+    # doesn't cover.
+    def instructions
+      return convention_instructions if rule.convention
+
+      text = rule.window? ? "Look at lines #{first}-#{last} of the file; the rest is context. #{rule.question}" : rule.question
+      return text unless rule.fail_when == "yes"
+
+      scope = { "window" => "those lines do it", "file" => "the code does it", "directory" => "the files do it" }
+              .fetch(rule.scope)
+      "#{text} Answer yes only when #{scope}, not when the question doesn't apply."
+    end
+
+    def yes_means
+      return rule.yes_means unless rule.convention
+
+      @inverted ? rule.no_means || FOLLOWS : rule.yes_means || BREAKS
+    end
+
+    def no_means
+      return rule.no_means unless rule.convention
+
+      @inverted ? rule.yes_means || BREAKS : rule.no_means || FOLLOWS
+    end
+
+    def digest
+      @digest ||= Digest::SHA256.hexdigest(JSON.generate([instructions, yes_means.to_s, no_means.to_s]))
+    end
+
+    private
+
+    # The spec's prose is in the state as team_rules, so the bullet is "the
+    # following team rule".
+    def convention_instructions
+      ask =
+        case rule.scope
+        when "window"
+          if @inverted
+            "Do lines #{first}-#{last} of the file follow the following team rule, or does it not apply to them? " \
+              "Answer false only when those lines clearly break it."
+          else
+            "Do lines #{first}-#{last} of the file clearly violate the following team rule? Answer true only " \
+              "when the code breaks the rule, not when the rule does not apply to it."
+          end
+        when "file"
+          if @inverted
+            "Does the file follow the following team rule, or does it not apply to it? Answer false only when " \
+              "the file clearly breaks it."
+          else
+            "Does the file clearly violate the following team rule? Answer true only when the code breaks the " \
+              "rule, not when the rule does not apply to it."
+          end
+        else
+          if @inverted
+            "Do the files follow the following team rule, or does it not apply to them? Answer false only when " \
+              "they clearly break it."
+          else
+            "Do the files clearly violate the following team rule? Answer true only when the code breaks the " \
+              "rule, not when the rule does not apply to it."
+          end
+        end
+      "#{ask} Rule: #{rule.convention}"
+    end
+  end
+end

data/lib/judgelint/reporter.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Judgelint
+  # Prints results as text, JSON, GitHub Actions annotations, or SARIF. Data
+  # goes to stdout; anything else goes to stderr.
+  class Reporter
+    FORMATS = %w[text json github sarif].freeze
+    MARKS = { violation: "x", uncertain: "?", skipped: "-", refused: "-", suppressed: "-" }.freeze
+    STATUSES = %w[violation uncertain pass skipped refused suppressed].freeze
+
+    def initialize(out, format:)
+      @out = out
+      @format = format
+    end
+
+    def check(verdicts, config:, file_count:, usage:)
+      case @format
+      when "json" then json(check_json(verdicts, config, file_count, usage))
+      when "github" then verdicts.each { |verdict| annotate(verdict) }
+      when "sarif" then json(sarif(verdicts))
+      else text(verdicts, file_count)
+      end
+    end
+
+    def fixtures(examples, usage:)
+      return json(fixtures_json(examples, usage)) if @format == "json"
+
+      examples.group_by(&:rule).each do |rule, group|
+        correct = group.count(&:correct?)
+        @out.puts "#{rule.id}  #{correct}/#{group.size}"
+        group.each do |example|
+          mark = example.correct? ? "ok" : "MISS"
+          @out.puts format("  %-4s %-4s %s  %s", mark, example.expected, probability(example), example.path)
+        end
+      end
+      @out.puts "" unless examples.empty?
+      @out.puts "#{examples.count(&:correct?)}/#{examples.size} examples judged correctly"
+    end
+
+    private
+
+    def json(data)
+      @out.puts JSON.pretty_generate(data)
+    end
+
+    def text(verdicts, file_count)
+      shown = verdicts.reject { |verdict| verdict.status == :pass }
+      width = shown.map { |verdict| verdict.rule.id.size }.max
+      shown.group_by(&:path).each do |path, group|
+        @out.puts path
+        group.each { |verdict| @out.puts "  #{line(verdict, width)}" }
+      end
+      @out.puts "" unless shown.empty?
+      @out.puts summary(verdicts, file_count)
+    end
+
+    def line(verdict, width)
+      label = verdict.status == :violation ? verdict.rule.severity : verdict.status.to_s
+      format("%s %-10s %-#{width}s  %-7s %s  %s", MARKS.fetch(verdict.status), label, verdict.rule.id,
+             range(verdict), probability(verdict), verdict.rule.message)
+    end
+
+    def range(verdict)
+      verdict.lines ? verdict.lines.join("-") : ""
+    end
+
+    def probability(verdict)
+      return "  - " if verdict.probability.nil?
+
+      format("%.2f", verdict.probability)
+    end
+
+    def summary(verdicts, file_count)
+      counts = verdicts.group_by(&:status).transform_values(&:size)
+      parts = ["#{file_count} #{file_count == 1 ? 'file' : 'files'}", "#{counts.fetch(:violation, 0)} violations"]
+      parts << "#{counts[:uncertain]} uncertain" if counts[:uncertain]
+      parts << "#{counts[:skipped]} skipped (too large)" if counts[:skipped]
+      parts << "#{counts[:refused]} refused by the model" if counts[:refused]
+      parts << "#{counts[:suppressed]} suppressed" if counts[:suppressed]
+      parts.join(", ")
+    end
+
+    def check_json(verdicts, config, file_count, usage)
+      counts = verdicts.group_by(&:status).transform_values(&:size)
+      {
+        "version" => VERSION,
+        "models" => config.models.map(&:label),
+        "files" => file_count,
+        "summary" => STATUSES.to_h { |status| [status, counts.fetch(status.to_sym, 0)] },
+        "usage" => usage.to_h,
+        "results" => verdicts.map(&:to_h)
+      }
+    end
+
+    def fixtures_json(examples, usage)
+      {
+        "version" => VERSION,
+        "correct" => examples.count(&:correct?),
+        "total" => examples.size,
+        "usage" => usage.to_h,
+        "examples" => examples.map(&:to_h)
+      }
+    end
+
+    # GitHub Actions workflow commands, which show up on the pull request's lines.
+    def annotate(verdict)
+      level = { violation: verdict.rule.error? ? "error" : "warning", uncertain: "notice" }[verdict.status]
+      return unless level
+
+      properties = ["file=#{escape(verdict.path, true)}"]
+      properties += ["line=#{verdict.lines[0]}", "endLine=#{verdict.lines[1]}"] if verdict.lines
+      properties << "title=#{escape("judgelint #{verdict.rule.id}", true)}"
+      @out.puts "::#{level} #{properties.join(',')}::#{escape("#{verdict.rule.message} (#{probability(verdict)})", false)}"
+    end
+
+    def escape(text, property)
+      escaped = text.gsub("%", "%25").gsub("\r", "%0D").gsub("\n", "%0A")
+      property ? escaped.gsub(":", "%3A").gsub(",", "%2C") : escaped
+    end
+
+    def sarif(verdicts)
+      shown = verdicts.select { |verdict| %i[violation uncertain].include?(verdict.status) }
+      rules = shown.map(&:rule).uniq
+      {
+        "$schema" => "https://json.schemastore.org/sarif-2.1.0.json",
+        "version" => "2.1.0",
+        "runs" => [{
+          "tool" => { "driver" => {
+            "name" => "judgelint", "version" => VERSION, "informationUri" => "https://github.com/jpcamara/judgelint",
+            "rules" => rules.map { |rule| { "id" => rule.id, "shortDescription" => { "text" => rule.message } } }
+          } },
+          "results" => shown.map { |verdict| sarif_result(verdict) }
+        }]
+      }
+    end
+
+    def sarif_result(verdict)
+      region = verdict.lines ? { "startLine" => verdict.lines[0], "endLine" => verdict.lines[1] } : {}
+      level = verdict.status == :uncertain ? "note" : (verdict.rule.error? ? "error" : "warning")
+      {
+        "ruleId" => verdict.rule.id, "level" => level,
+        "message" => { "text" => "#{verdict.rule.message} (#{probability(verdict)})" },
+        "locations" => [{ "physicalLocation" => { "artifactLocation" => { "uri" => verdict.path }, "region" => region } }]
+      }
+    end
+  end
+end

data/lib/judgelint/rule.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Judgelint
+  # One convention, asked of a file as a single yes/no question. Rules are
+  # markdown files with YAML frontmatter:
+  #
+  #   ---
+  #   question: Does this code rescue an exception and carry on as if nothing happened?
+  #   yes_means: The rescue neither logs, re-raises, nor returns an explicit failure.
+  #   no_means: Every rescue logs, re-raises, or turns the error into a result.
+  #   paths: ["app/**/*.rb", "lib/**/*.rb"]
+  #   ---
+  #   Swallowed exceptions hide bugs until they cost us data.
+  #
+  # The body is the rationale people read. The model sees only the question
+  # and the meanings of yes and no. +fail_when+ is the answer that means the
+  # file breaks the convention, so a rule can ask "Does this rescue and discard
+  # an exception?" (fail on yes) or "Does this service validate its input
+  # first?" (fail on no).
+  #
+  # +scope+ "window" asks about each range of lines, so a finding says where
+  # it is; "file" asks once about the whole file, for conventions no range can
+  # show on its own; "directory" asks once about each directory its paths
+  # match, with all of the directory's files, for conventions about how files
+  # relate. A fail_when: no rule is about the whole file by default.
+  #
+  # +context+ globs name files the model reads alongside, such as shared
+  # types, which are never judged themselves.
+  class Rule
+    FRONTMATTER = /\A---[ \t]*\r?\n(.*?)\r?\n---[ \t]*(?:\r?\n|\z)(.*)\z/m
+    KEYS = %w[id question convention yes_means no_means fail_when scope paths excludes context severity threshold message].freeze
+
+    attr_reader :id, :question, :convention, :yes_means, :no_means, :fail_when, :scope, :paths, :excludes,
+                :context, :severity, :threshold, :message, :source, :examples_dir, :prose
+
+    # The rules in +dir+ alone (Spec.discover also finds READMEs).
+    def self.load_dir(dir, default_threshold:)
+      Spec.discover(rules_dir: dir, default_threshold: default_threshold, readmes: false)
+    end
+
+    # A rule is a question asked exactly as written, or a convention (a
+    # spec's bullet) asked whether the code breaks it.
+    # +prose+ is a spec's body, which the model reads alongside the file.
+    def initialize(data, source:, body: "", default_threshold: 0.7, examples_dir: nil, prose: nil)
+      unknown = data.keys - KEYS
+      raise ConfigError, "#{source}: unknown keys #{unknown.join(', ')}" unless unknown.empty?
+
+      @source = source
+      @id = (data["id"] || File.basename(source, ".md")).to_s
+      @question = text(data["question"])
+      @convention = text(data["convention"])
+      raise ConfigError, "#{source}: a rule needs a question" if @question.nil? && @convention.nil?
+
+      @examples_dir = examples_dir || source.delete_suffix(".md")
+      @prose = prose
+      @yes_means = text(data["yes_means"])
+      @no_means = text(data["no_means"])
+      @fail_when = one_of("fail_when", yes_or_no(data.fetch("fail_when", "yes")), %w[yes no])
+      @scope = one_of("scope", data.fetch("scope", @fail_when == "yes" ? "window" : "file"), %w[window file directory])
+      @severity = one_of("severity", data.fetch("severity", "error"), %w[error warning])
+      @paths = list(data["paths"], ["**/*"])
+      @excludes = list(data["excludes"], [])
+      @context = list(data["context"], [])
+      @threshold = parse_threshold(data.key?("threshold") ? data["threshold"] : default_threshold)
+      @message = (data["message"] || first_paragraph(body) || @question || @convention).to_s.strip
+    end
+
+    def applies_to?(path)
+      Glob.match_any?(paths, path) && !Glob.match_any?(excludes, path)
+    end
+
+    def question_key
+      id.gsub(/[^A-Za-z0-9_]/, "_")
+    end
+
+    def window?
+      scope == "window"
+    end
+
+    def directory?
+      scope == "directory"
+    end
+
+    def context_files
+      Glob.expand(context)
+    end
+
+    # The directories a directory rule governs.
+    def directories
+      paths.flat_map { |pattern| Dir.glob(pattern) }.select { |path| File.directory?(path) && applies_to?(path) }.uniq.sort
+    end
+
+    # The ranges to ask about: windows of +size+ lines, or the whole file.
+    def ranges(source, size)
+      window? ? source.windows(size) : [[1, source.line_count]].reject { |_first, last| last.zero? }
+    end
+
+    def violation_probability(yes_probability)
+      fail_when == "yes" ? yes_probability : 1.0 - yes_probability
+    end
+
+    def error?
+      severity == "error"
+    end
+
+    private
+
+    # YAML reads a bare yes/no as true/false.
+    def yes_or_no(value)
+      return "yes" if value == true
+      return "no" if value == false
+
+      value
+    end
+
+    def one_of(key, value, allowed)
+      value = value.to_s
+      raise ConfigError, "#{source}: #{key} is #{allowed.join(' or ')}, not #{value}" unless allowed.include?(value)
+
+      value
+    end
+
+    def parse_threshold(value)
+      number = Judgelint.number(value)
+      raise ConfigError, "#{source}: threshold must be a number, not #{value}" if number.nil?
+      raise ConfigError, "#{source}: threshold must be between 0 and 1" unless number.positive? && number <= 1
+
+      number
+    end
+
+    def list(value, default)
+      return default if value.nil?
+
+      value.is_a?(Array) ? value.map(&:to_s) : [value.to_s]
+    end
+
+    def text(value)
+      string = value.to_s.strip
+      string.empty? ? nil : string
+    end
+
+    def first_paragraph(body)
+      paragraph = body.split(/\n\s*\n/).map(&:strip).find { |part| !part.empty? && !part.start_with?("#") }
+      paragraph&.gsub(/\s*\n\s*/, " ")
+    end
+  end
+end

data/lib/judgelint/runner.rb

@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+module Judgelint
+  # Judges files against the rules that govern them, and rules against their
+  # examples, in two steps: plan every request first, so a run can be priced
+  # and refused before anything is sent, then send them.
+  #
+  # Each file's requests carry the whole numbered file as state and one
+  # question per rule and window, so a file's verdicts never depend on which
+  # other files ran alongside it. Every model is asked every question, all of a
+  # file's requests go out at once, and a question's answer is the models'
+  # average. A rule's verdict is its worst window.
+  class Runner
+    # One file's work: the questions to ask, each model's answers (from the
+    # cache or the model), and the requests still to send.
+    class Job
+      attr_reader :source, :state, :state_digest, :questions, :suppressed, :answers, :requests, :expected
+      attr_accessor :skipped
+
+      # The state is built and hashed once: it's the whole file, and every
+      # question's cache key needs its hash. Rules from a spec read its prose
+      # alongside the file, as team_rules.
+      def initialize(source, questions, suppressed, prose: nil, expected: nil)
+        @source = source
+        @state = prose ? { "team_rules" => prose, "file" => source.state } : source.state
+        @state_digest = Digest::SHA256.hexdigest(JSON.generate(@state))
+        @questions = questions
+        @suppressed = suppressed
+        @answers = {} # model index => {question key => probability or nil}
+        @requests = []
+        @expected = expected
+        @skipped = false
+      end
+    end
+
+    # Questions sent together to one model, and their estimated input tokens.
+    class Request
+      attr_reader :model, :questions, :tokens
+
+      def initialize(model, questions, tokens)
+        @model = model
+        @questions = questions
+        @tokens = tokens
+      end
+    end
+
+    # +clients+ has one Client per model in config.models.
+    def initialize(config:, rules:, clients:, cache:)
+      @config = config
+      @rules = rules
+      @clients = clients
+      @cache = cache
+    end
+
+    # +changes+ ({path => lines or :all}, from Changes.since) limits each file
+    # to the windows a change touched, and directories to those with a change.
+    # Rules that read different context files can't share a request, so a
+    # file gets one job per set of context files.
+    def plan(paths, directories: [], changes: nil)
+      files = paths.flat_map do |path|
+        rules = @rules.select { |rule| !rule.directory? && rule.applies_to?(path) }
+        by_context(rules) { |context, group| plan_file(Source.new(path, context: context), group, changes ? changes[path] : :all) }
+      end
+      units = directories.flat_map do |dir|
+        next [] if changes && changes.keys.none? { |path| path.start_with?("#{dir}/") }
+
+        rules = @rules.select { |rule| rule.directory? && rule.applies_to?(dir) }
+        by_context(rules) { |context, group| plan_file(Source.directory(dir, context: context), group, :all) }
+      end
+      files + units
+    end
+
+    attr_reader :untested
+
+    # Files under <examples>/pass must not be judged violations, files under
+    # <examples>/fail must. The model sees each example's path relative to its
+    # pass/ or fail/ directory, so the directory never gives the answer away.
+    # A rule with examples needs both kinds, checked before anything is sent;
+    # rules with none are listed in +untested+.
+    def plan_examples
+      jobs = []
+      missing = []
+      @untested = []
+      @rules.each do |rule|
+        found = %i[pass fail].to_h { |expected| [expected, example_paths(rule, expected)] }
+        if found.values.all?(&:empty?)
+          @untested << rule
+          next
+        end
+
+        found.each do |expected, paths|
+          dir = File.join(rule.examples_dir, expected.to_s)
+          missing << "#{dir}/" if paths.empty?
+          paths.each do |path|
+            shown = path.delete_prefix("#{dir}/")
+            context = rule.context_files
+            source = rule.directory? ? Source.directory(path, shown_as: shown, context: context)
+                                     : Source.new(path, shown_as: shown, context: context)
+            jobs << plan_file(source, [rule], :all, expected: expected)
+          end
+        end
+      end
+      raise ConfigError, "Examples need both kinds: add some to #{missing.join(', ')}" unless missing.empty?
+
+      jobs
+    end
+
+    def request_count(jobs)
+      jobs.sum { |job| job.requests.size }
+    end
+
+    def estimated_tokens(jobs)
+      jobs.sum { |job| job.requests.sum(&:tokens) }
+    end
+
+    # Nil when a model has no price.
+    def estimated_cost(jobs)
+      requests = jobs.flat_map(&:requests)
+      prices = requests.map { |request| @clients[request.model].endpoint.price_per_million }
+      return nil if prices.include?(nil)
+
+      requests.each_with_index.sum { |request, index| request.tokens * prices[index] / 1_000_000.0 }
+    end
+
+    # Returns each job's verdicts, in the jobs' order.
+    def execute(jobs)
+      Pool.map(jobs, Config::THREADS) do |job|
+        ask(job)
+        verdicts(job)
+      end
+    ensure
+      @cache.save
+    end
+
+    private
+
+    # A directory rule's examples are the directories under pass/ and fail/.
+    def example_paths(rule, expected)
+      dir = File.join(rule.examples_dir, expected.to_s)
+      return Dir.glob(File.join(dir, "*")).select { |path| File.directory?(path) }.sort if rule.directory?
+
+      Glob.expand([File.join(dir, "**", "*")])
+    end
+
+    def plan_file(source, rules, lines, expected: nil)
+      unknown = source.suppressed_ids - @rules.map(&:id)
+      raise ConfigError, "#{source.path}: judgelint:disable names unknown rules #{unknown.join(', ')}" unless unknown.empty?
+
+      questions = {}
+      suppressed = []
+      rules.each do |rule|
+        ranges = rule.ranges(source, Config::WINDOW_LINES).select { |first, last| touched?(lines, first, last) }
+        next if ranges.empty?
+
+        kept = ranges.reject { |first, last| source.suppressed?(rule.id, first, last) }
+        if kept.empty?
+          suppressed << rule
+        else
+          asked = []
+          kept.each do |first, last|
+            asked << Question.new(rule, first, last)
+            asked << Question.new(rule, first, last, inverted: true) if rule.convention
+          end
+          questions[rule] = asked
+        end
+      end
+      job = Job.new(source, questions, suppressed, prose: rules.first&.prose, expected: expected)
+      pack(job)
+      job
+    end
+
+    # Rules that read different context files or spec prose can't share a request.
+    def by_context(rules)
+      rules.group_by { |rule| [rule.context, rule.prose] }.map { |(globs, _prose), group| yield(Glob.expand(globs), group) }
+    end
+
+    def touched?(lines, first, last)
+      lines == :all || (lines || []).any? { |line| line.between?(first, last) }
+    end
+
+    # For each model, packs the questions its cache can't answer into as few
+    # requests as fit its limits. A file whose state and widest question are
+    # over a model's limit is skipped: it's reported, never sent.
+    def pack(job)
+      @clients.each_with_index do |client, model|
+        answers = job.answers[model] = {}
+        pending = []
+        job.questions.each_value do |questions|
+          questions.each do |question|
+            cached = @cache.fetch(cache_key(question, job, client))
+            cached.nil? ? pending << question : answers[question.key] = cached
+          end
+        end
+        next if pending.empty?
+
+        limits = client.endpoint
+        base = client.tokens(job.state, [])
+        sizes = pending.map { |question| client.question_tokens(question) }
+        if base + sizes.max >= limits.max_state_tokens
+          job.skipped = true
+          job.requests.clear
+          return
+        end
+
+        batch = []
+        tokens = base
+        pending.each_with_index do |question, index|
+          if !batch.empty? && (tokens + sizes[index] > limits.max_request_tokens || batch.size >= limits.batch_size)
+            job.requests << Request.new(model, batch, tokens)
+            batch = []
+            tokens = base
+          end
+          batch << question
+          tokens += sizes[index]
+        end
+        job.requests << Request.new(model, batch, tokens)
+      end
+    end
+
+    # Sends every request at once: a file's packed requests and every model's.
+    def ask(job)
+      state = job.state.dup # Spinel can't yet pass the Job's shared String along
+      results = Pool.map(job.requests, job.requests.size) do |request|
+        @clients[request.model].judge(state, request.questions)
+      end
+      job.requests.each_with_index do |request, index|
+        client = @clients[request.model]
+        request.questions.each do |question|
+          probability = results[index][question.key]
+          @cache.store(cache_key(question, job, client), probability) unless probability.nil?
+          job.answers[request.model][question.key] = probability
+        end
+      end
+    end
+
+    def verdicts(job)
+      path = job.source.path
+      found = job.questions.map { |rule, questions| verdict_for(rule, path, questions, job) }
+      found += job.suppressed.map { |rule| Verdict.new(rule: rule, path: path, probability: nil, status: :suppressed) }
+      job.expected ? found.map { |verdict| verdict.as_example(job.expected) } : found
+    end
+
+    # A window's probability of a violation is the average over its questions
+    # (a bullet is asked both ways) and the models; a rule's verdict is its
+    # worst window.
+    def verdict_for(rule, path, questions, job)
+      return Verdict.new(rule: rule, path: path, probability: nil, status: :skipped) if job.skipped
+
+      windows = []
+      questions.group_by { |question| [question.first, question.last] }.each do |(first, last), asked|
+        values = []
+        asked.each do |question|
+          job.answers.each_value do |answers|
+            yes = answers[question.key]
+            values << question.violation(yes) unless yes.nil?
+          end
+        end
+        windows << [first, last, values.sum / values.size] unless values.empty?
+      end
+      return Verdict.new(rule: rule, path: path, probability: nil, status: :refused) if windows.empty?
+
+      first, last, probability = windows.max_by { |window| window[2] }
+      lines = rule.directory? ? nil : [first, last]
+      Verdict.classify(rule, path, probability, warn_at: @config.warn_at, lines: lines)
+    end
+
+    # Each model's answers are cached on their own, so adding a model asks
+    # only the new one.
+    def cache_key(question, job, client)
+      endpoint = client.endpoint
+      Cache.key(question.digest, job.state_digest, "#{endpoint.provider}/#{endpoint.model}")
+    end
+  end
+end