sloplint-judge 0.1.0

data/lib/sloplint/judge/backends/jev.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "openssl"
+require "uri"
+require "json"
+require_relative "../secret"
+
+module Sloplint
+  module Judge
+    module Backends
+      # Jev, TypeSafe's System One model. The rule's question shape is Jev's,
+      # so questions pass through verbatim.
+      class Jev
+        DEFAULT_URL = "https://api.typesafe.ai/v1/systemone"
+        DEFAULT_MODEL = "jev-latest"
+        # The environment variable, and the keychain account, this backend's
+        # key lives under. Another backend declares its own, so two keys are
+        # two items.
+        KEY = "TYPESAFE_API_KEY"
+
+        # key: nil means look it up (environment, then keychain; see Secret).
+        def initialize(url: ENV.fetch("SYSTEMONE_URL", DEFAULT_URL),
+                       model: ENV.fetch("SYSTEMONE_MODEL", DEFAULT_MODEL),
+                       key: nil)
+          # The URL first: reading the key means the environment and then a
+          # keychain subprocess, which can sit for fifteen seconds on a
+          # dialog nobody can see, and a run that is about to be refused for
+          # its URL has no business pulling the secret out of the keychain.
+          @url = self.class.https!(url)
+          key ||= Secret.fetch(KEY)&.first
+          raise ArgumentError, Secret.missing(KEY) if key.nil? || key.empty?
+
+          @model = model
+          @key = key
+        end
+
+        # The name a run would report, without building the backend or
+        # reading the key, so a run that asks nothing still names the model
+        # the way a run that asks something does.
+        def self.model_name = ENV.fetch("SYSTEMONE_MODEL", DEFAULT_MODEL)
+
+        # The endpoint settings, checked without building the backend or
+        # reading the key; `status` prints what this returns.
+        def self.configured!
+          https!(ENV.fetch("SYSTEMONE_URL", DEFAULT_URL))
+          "model #{model_name}"
+        end
+
+        # Case-insensitive, because a DNS name is: API.TypeSafe.ai is the
+        # same host as api.typesafe.ai, and URI hands back the host as it was
+        # written. The scheme needs no such care -- URI lowercases that one.
+        HOST = /\A(?:.+\.)?typesafe\.ai\z/i
+
+        # The key and the whole document go to this URL, so it is https and
+        # a TypeSafe host or nothing. The host pin is what keeps an injected
+        # `SYSTEMONE_URL=https://attacker.example sloplint check --judge` from
+        # being a valid way to run the sanctioned command.
+        def self.https!(url)
+          # URI raises its own error class, which the commands do not catch:
+          # a misspelt SYSTEMONE_URL would end `status` with a backtrace
+          # instead of the usage error every other bad setting gets.
+          uri = begin
+            URI(url)
+          rescue URI::InvalidURIError
+            raise ArgumentError, "SYSTEMONE_URL is not a URL: #{url.inspect}"
+          end
+          raise ArgumentError, "SYSTEMONE_URL must be https, got #{uri.scheme.inspect}" unless uri.scheme == "https"
+          raise ArgumentError, "SYSTEMONE_URL must be a typesafe.ai host, got #{uri.host.inspect}" unless uri.host.to_s.match?(HOST)
+          raise ArgumentError, "SYSTEMONE_URL needs a path, for example #{DEFAULT_URL}" if uri.path.empty?
+
+          uri
+        end
+
+        # Jev's response carries token counts and no price. TypeSafe's public
+        # price is $42 per billion input tokens; output tokens are free. So
+        # the cost is the input tokens times this, exactly, not an estimate.
+        # ponytail: one constant, not a config; move to an env var when the
+        # price moves.
+        USD_PER_INPUT_TOKEN = 42.0 / 1_000_000_000
+
+        def name = @model
+
+        # Dollars for a summed usage Hash. Output tokens cost nothing. On the
+        # class as well, because the run that asks nothing has no built
+        # backend to ask and still prints a price.
+        def self.cost_usd(usage) = usage.fetch("input_tokens", 0) * USD_PER_INPUT_TOKEN
+        def cost_usd(usage) = self.class.cost_usd(usage)
+
+        def ask(state, questions)
+          http = Net::HTTP.new(@url.host, @url.port)
+          http.use_ssl = true
+          http.read_timeout = 90
+          req = Net::HTTP::Post.new(@url.request_uri, "Content-Type" => "application/json", "Authorization" => "Bearer #{@key}")
+          req.body = JSON.generate({ "state" => state, "model" => @model, "questions" => questions })
+          res = http.request(req)
+          raise BackendError, "#{@url.host} returned #{res.code}: #{res.body.to_s[0, 200]}" unless res.code == "200"
+
+          body = JSON.parse(res.body)
+          # A body of [] or null reads as an answer until something is asked
+          # of it. Said here, it is a malformed body like any other; left to
+          # the reads below it would be a TypeError, which is a bug's class.
+          malformed!("the body is #{body.class}, not a Hash") unless body.is_a?(Hash)
+
+          usage = body["usage"].is_a?(Hash) ? body["usage"] : {}
+          answers = body.fetch("answers")
+          malformed!("answers is #{answers.class}, not a Hash keyed by question") unless answers.is_a?(Hash)
+          questions.to_h do |qname, q|
+            a = answers.fetch(qname) { raise KeyError, "no answer for #{qname}" }
+            malformed!("the answer for #{qname} is #{a.class}, not a Hash") unless a.is_a?(Hash)
+            type = q.fetch("type")
+            probs = normalise(type, type == "noul" ? a.fetch("noul") : a.fetch("probabilities"), q)
+            [qname, Answer.new(type: type, probabilities: probs, confidence: confidence_for(a, type, probs), usage: usage)]
+          end
+        # Transport errors, and a body missing a key Jev documents, are one
+        # thing to the caller: the backend did not answer. Exit 3, not a
+        # backtrace. A body of the wrong shape raises BackendError where it is
+        # read, so this list stays off our own code: a bug in normalise is a
+        # bug, and it gets to raise like one.
+        # Net::HTTPBadResponse and Net::HTTPHeaderSyntaxError are named one by
+        # one because they descend from StandardError and not from
+        # Net::ProtocolError: a proxy that answers with something that is not
+        # an HTTP status line raises one of them.
+        rescue SocketError, IOError, SystemCallError, OpenSSL::SSL::SSLError, Net::ProtocolError,
+               Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::OpenTimeout,
+               Net::ReadTimeout, Net::WriteTimeout, JSON::ParserError, KeyError => e
+          raise BackendError, "#{e.class}: #{e.message}"
+        end
+
+        private
+
+        # Jev returns score probabilities keyed by level index as strings, choice
+        # probabilities keyed by option, and a noul as one float under "noul".
+        # Anything else in those slots is a malformed body, said so here, which
+        # is what lets `ask` rescue only the reads and not our own arithmetic.
+        def normalise(type, probs, question)
+          case type
+          when "score" then scores(probs, question.fetch("criteria").size)
+          when "choice"
+            malformed!("choice probabilities are #{probs.class}, not a Hash keyed by option") unless probs.is_a?(Hash)
+
+            probs.transform_values { |p| number(p) }
+          else
+            malformed!("noul is #{probs.class}, not a number") unless probs.is_a?(Numeric)
+
+            probs.to_f
+          end
+        end
+
+        # One probability per level, in level order. Jev may leave out a level
+        # whose probability is zero, so the array is as long as the question's
+        # criteria and the gaps are 0.0. Otherwise a short array would slide
+        # every level down one and Answer#top would name the wrong criterion.
+        def scores(probs, levels)
+          malformed!("score probabilities are #{probs.class}, not a Hash keyed by level") unless probs.is_a?(Hash)
+          # No level at all is not a level of zero. Filled with zeros the
+          # levels would tie, the tie would go to level 0, and level 0 is the
+          # one every score rule flags on.
+          malformed!("score probabilities for none of the #{levels} levels") if probs.empty?
+
+          out = Array.new(levels, 0.0)
+          probs.each do |k, p|
+            i = Integer(k.to_s, 10, exception: false)
+            malformed!("score probability key #{k.inspect} is not one of the #{levels} levels") unless i&.between?(0, levels - 1)
+            out[i] = number(p)
+          end
+          out
+        end
+
+        def number(p) = Float(p, exception: false) || malformed!("#{p.class} is not a probability")
+
+        def malformed!(why) = raise(BackendError, "#{@url.host} returned a malformed body: #{why}")
+
+        # Score and choice answers carry a confidence. A noul does not, so it
+        # gets the one thing its answer can defend: distance from the fence,
+        # scaled so 0.5 is 0 and 0 or 1 is 1.
+        def confidence_for(answer, type, probs)
+          # Read like any other number Jev sends: a confidence of {} or [] or
+          # true is a malformed body, not a NoMethodError on our own side.
+          return number(answer["confidence"]) if answer["confidence"]
+          return (probs - 0.5).abs * 2 if type == "noul"
+
+          sorted = (probs.is_a?(Hash) ? probs.values : probs).sort.reverse
+          malformed!("an answer with no probabilities at all") if sorted.empty?
+
+          (sorted[0] - (sorted[1] || 0)).clamp(0.0, 1.0)
+        end
+      end
+    end
+  end
+end

data/lib/sloplint/judge/cli.rb

@@ -0,0 +1,290 @@
+# frozen_string_literal: true
+
+require "optparse"
+require "json"
+require "sloplint/judge"
+
+module Sloplint
+  module Judge
+    # The judge's own executable: judge-only check, compare, rules, explain.
+    # Exit codes are sloplint's plus 3 for a backend failure. See docs/JUDGE.md.
+    module CLI
+      module_function
+
+      def run(argv, out: $stdout, err: $stderr, stdin: $stdin)
+        opts = { format: "full", register: Engine::DEFAULT_REGISTER }
+        parser = global_parser(opts, out:)
+        begin
+          parser.order!(argv)
+        rescue OptionParser::InvalidOption => e
+          argv.unshift("check", *e.args)
+        end
+        return 0 if opts[:help_shown] || opts[:version_shown]
+
+        command = argv.shift
+        command ||= (stdin.tty? ? "help" : "check")
+        case command
+        when "check"   then cmd_check(argv, opts, out:, err:, stdin:)
+        when "compare" then cmd_compare(argv, opts, out:, err:)
+        when "rules"   then cmd_rules(argv, out:)
+        when "explain" then cmd_explain(argv, out:, err:)
+        when "status"  then cmd_status(opts, out:, err:)
+        when "key"     then cmd_key(argv, opts, out:, err:, stdin:)
+        when "version" then out.puts(VERSION); 0
+        when "help"    then out.puts(parser.help); 0
+        else cmd_check(argv.unshift(command), opts, out:, err:, stdin:)
+        end
+      rescue OptionParser::ParseError => e
+        err.puts("sloplint-judge: #{e.message}")
+        2
+      rescue BackendError => e
+        err.puts("sloplint-judge: backend failure: #{e.message}")
+        3
+      end
+
+      # ── check ───────────────────────────────────────────────────────────────
+      def cmd_check(argv, opts, out:, err:, stdin:)
+        markdown = false
+        strict = false
+        select = nil
+        ignore = nil
+        help = false
+        OptionParser.new do |o|
+          o.banner = "usage: sloplint-judge check [options] [paths...]  (\"-\" or no paths = stdin)"
+          o.on("-o", "--output-format FORMAT", %w[full json], "Output format: 'full' or 'json' (default: full).") { |v| opts[:format] = v }
+          o.on("--markdown", "Skip fenced/inline code spans, HTML comments, URLs and Markdown furniture.") { markdown = true }
+          o.on("--select IDS", "Only run these rules (comma-separated rule ids or categories).") { |v| select = v.split(",").map(&:strip) }
+          o.on("--ignore IDS", "Skip these rules (comma-separated rule ids or categories).") { |v| ignore = v.split(",").map(&:strip) }
+          o.on("--strict", "Run the rules that are off by default, ask the sentence rules about",
+               "every sentence, and keep low-confidence notes; about three times the",
+               "requests of a default run.") { strict = true }
+          o.on("--register TEXT", "Who the reader is.") { |v| opts[:register] = v }
+          o.on("--backend NAME", "Which adapter to use.") { |v| opts[:backend] = v }
+          # OptionParser answers -h itself when nobody else does, and it
+          # answers it on the real stdout and ends the process. This command
+          # prints to the out it was given and returns, as every other one does.
+          o.on("-h", "--help", "Show this help.") { out.puts(o.help); help = true }
+        # permute!, so a flag written after the path is a flag: `sloplint-judge
+        # check draft.md --markdown` reads the way anyone would write it.
+        end.permute!(argv)
+        return 0 if help
+
+        unknown = Sloplint::CLI.unknown_rule_refs(select, RULES) + Sloplint::CLI.unknown_rule_refs(ignore, RULES)
+        unless unknown.empty?
+          err.puts("sloplint-judge: unknown rule or category: #{unknown.join(", ")}")
+          err.puts("run `sloplint-judge rules` to list them.")
+          return 2
+        end
+        rules = Sloplint::CLI.select_rules(select, ignore, strict, RULES)
+
+        # Only the read is invalid input. An ArgumentError out of the run is
+        # a bug in this code, and it raises like one instead of sending the
+        # reader to look for bad bytes in a file that has none.
+        sources = begin
+          Sloplint::CLI.read_sources(argv.empty? ? ["-"] : argv, err:, stdin:, name: "sloplint-judge")
+        rescue ArgumentError, Encoding::CompatibilityError => e
+          err.puts("sloplint-judge: invalid input: #{e.message}")
+          return 2
+        end
+        return 2 unless sources
+
+        # A selection that holds no rule is decided here, before a backend is
+        # built, because building one reads the key: `--ignore` every rule
+        # must run on a machine that has none. Nothing was asked, so nothing
+        # was found, and that is exit 0 with a 0-request usage line -- not
+        # the "nothing examined" error below, which is a document that had no
+        # prose in it.
+        if rules.empty?
+          usage = begin
+            Engine.nothing_asked(opts[:backend])
+          rescue ArgumentError => e
+            err.puts("sloplint-judge: #{e.message}")
+            return 2
+          end
+          err.puts("sloplint-judge #{Engine.usage_line(usage)}")
+          Sloplint::CLI.emit([], opts[:format], out:, by_path: false, judge: usage)
+          return 0
+        end
+
+        backend = load_backend(opts[:backend], err:) or return 2
+        result = Engine.scan_sources(sources, name: "sloplint-judge", err:, rules:, backend:, markdown:, register: opts[:register], strict:)
+        # Nothing asked is nothing examined, and exit 0 would report that the
+        # judge read the document and found nothing in it. The same reason
+        # empty input is a usage error, and the same exit code.
+        if result.usage["requests"].zero?
+          names = sources.map { |label, _| label == "-" ? "stdin" : label }
+          err.puts("sloplint-judge: nothing examined in #{names.join(", ")}: no prose reached the judge.")
+          err.puts("a document that is all Markdown furniture, or whose paragraphs are too short for the rules selected, leaves nothing to ask about.")
+          return 2
+        end
+
+        Sloplint::CLI.emit(result.notes, opts[:format], out:, by_path: sources.count { |label, _| label != "-" } > 1, judge: result.usage)
+        result.notes.empty? ? 0 : 1
+      end
+
+      # ── compare A B ─────────────────────────────────────────────────────────
+      def cmd_compare(argv, opts, out:, err:)
+        drift = false
+        help = false
+        OptionParser.new do |o|
+          o.banner = "usage: sloplint-judge compare [--drift] A B  (two files)"
+          o.on("--drift", "Also ask whether B changes what A says.") { drift = true }
+          o.on("--register TEXT", "Who the reader is.") { |v| opts[:register] = v }
+          o.on("--backend NAME", "Which adapter to use.") { |v| opts[:backend] = v }
+          o.on("-h", "--help", "Show this help.") { out.puts(o.help); help = true }
+        # permute!, so `compare A B --drift` sees the flag. There is no nested
+        # subcommand here whose own flags an option after the files could be.
+        end.permute!(argv)
+        return 0 if help
+
+        a, b = argv
+        unless argv.size == 2 && File.file?(a) && File.file?(b)
+          err.puts("usage: sloplint-judge compare [--drift] A B")
+          return 2
+        end
+        # The same read `check` does, so an unreadable or badly encoded file
+        # is an invalid input here too, before a request is paid for.
+        sources = Sloplint::CLI.read_sources([a, b], err:, name: "sloplint-judge") or return 2
+
+        backend = load_backend(opts[:backend], err:) or return 2
+        verdict = Compare.run(sources[0].last, sources[1].last, place: "for #{opts[:register]}", backend:, drift:)
+        out.puts(JSON.pretty_generate(verdict.to_h.except(:usage)))
+        # Engine.numbers for the same reason a scan uses it: a count that is
+        # not a number must not take down a verdict already printed.
+        usage = Engine.with_cost({ "backend" => backend.name }.merge(Engine.numbers(verdict.usage)), backend)
+        err.puts("sloplint-judge: #{Engine.usage_line(usage)}")
+        0
+      end
+
+      # An unknown backend name or a missing key is a usage error (exit 2),
+      # not a backend failure (exit 3): nothing was tried.
+      def load_backend(name, err:)
+        Backend.load(name)
+      rescue ArgumentError => e
+        err.puts("sloplint-judge: #{e.message}")
+        nil
+      end
+
+      # ── status / key set ────────────────────────────────────────────────────
+      # Could a run happen here? Answered without reading the key: this is the
+      # check skill's probe and it runs before anyone has agreed to send text.
+      # Exit 0 configured, 2 not, with the message `check` would give.
+      def cmd_status(opts, out:, err:)
+        klass = Backend.klass(opts[:backend])
+        settings = klass.configured!
+        source = Secret.present?(klass::KEY) or raise ArgumentError, Secret.missing(klass::KEY)
+        out.puts("backend #{Backend.default_name(opts[:backend])} (#{settings}), key from #{source}")
+        0
+      rescue ArgumentError => e
+        err.puts("sloplint-judge: #{e.message}")
+        2
+      end
+
+      # Stores the key once. The platform tool owns the prompt and the
+      # terminal, so the value is never on a command line, in a pipe, in shell
+      # history or in this process. A person types this; the skill never does.
+      def cmd_key(argv, opts, out:, err:, stdin:)
+        key = Backend.klass(opts[:backend])::KEY
+        return cmd_key_unset(key, out:, err:) if argv == ["unset"]
+        return err.puts("usage: sloplint-judge [--backend NAME] key set|unset") || 2 unless argv == ["set"]
+
+        command = Secret.store_command(key)
+        return err.puts("sloplint-judge: no supported key store on this platform; export #{key} instead") || 2 unless command
+        return err.puts("sloplint-judge: key set needs a terminal: the keychain tool prompts for the key itself") || 2 unless stdin.tty?
+
+        out.puts("Storing #{key} in the OS keychain under #{Secret::SERVICE}; the keychain tool will prompt for it.")
+        out.puts("Replacing the item stored earlier.") if Secret.stored?(key)
+        out.puts("Any process running as you can read it back. This keeps the key out of dotfiles and out of the agent's environment, not out of your account.")
+        out.flush
+        Process.exec(*command)
+      rescue ArgumentError => e
+        err.puts("sloplint-judge: #{e.message}")
+        2
+      end
+
+      # Removes the keychain item. Says so when there was none, and says when a
+      # key in the environment is still there, because that one it cannot touch.
+      def cmd_key_unset(key, out:, err:)
+        command = Secret.delete_command(key)
+        return err.puts("sloplint-judge: no supported key store on this platform") || 2 unless command
+        return err.puts("sloplint-judge: no #{key} item in the keychain") || 2 unless Secret.stored?(key)
+
+        removed = system(*command, out: File::NULL, err: File::NULL)
+        return err.puts("sloplint-judge: the keychain tool could not remove the item") || 2 unless removed
+
+        out.puts("Removed #{key} from the OS keychain (#{Secret::SERVICE}).")
+        out.puts("It is still set in this shell's environment; unset it there too.") unless ENV[key].to_s.empty?
+        0
+      end
+
+      # ── rules / explain ─────────────────────────────────────────────────────
+      def cmd_rules(argv, out:)
+        as_json = false
+        OptionParser.new { |o| o.on("--json") { as_json = true } }.order!(argv)
+        Sloplint::CLI.render_rules(RULES, json: as_json, out:)
+        0
+      end
+
+      def cmd_explain(argv, out:, err:)
+        id = argv.shift
+        rule = RULES.find { |r| r.id == id }
+        unless rule
+          err.puts(id ? "sloplint-judge: no such rule: #{id}" : "usage: sloplint-judge explain RULE_ID")
+          return 2
+        end
+        levels = rule.question["criteria"].each_with_index.map do |c, i|
+          "  #{i}#{i == rule.flag[:level] ? " (flags)" : ""}: #{c["what"]}"
+        end
+        out.puts(<<~TXT)
+          #{rule.id}  (#{rule.category}, #{rule.severity}, #{rule.confidence} confidence, per #{rule.unit})
+
+          #{rule.message}
+
+          Asks: #{format(rule.question["instructions"], register: Engine::DEFAULT_REGISTER)}
+          #{levels.join("\n")}
+
+          Why: #{rule.rationale}
+          Fix: #{rule.suggestion}
+
+          Flags:    #{Sloplint::CLI.fixture_list(rule.examples_bad)}
+          Does not: #{Sloplint::CLI.fixture_list(rule.examples_ok)}
+        TXT
+        0
+      end
+
+      def global_parser(opts, out:)
+        OptionParser.new do |o|
+          o.banner = <<~BANNER
+            sloplint-judge — rules that ask a System One model what a regex cannot.
+
+            # Recommended for agents (both linters, one document). The judge sends the text to
+            # api.typesafe.ai, so ask the person before running it. Check first, ask, then run:
+            sloplint-judge status                            # exit 0 = a key is set up (reads no key), 2 = not
+            sloplint check --judge --markdown -o json FILE
+            # exit 0 = clean, 1 = notes found, 2 = usage error or not configured, 3 = the model could not be reached (nothing checked)
+            # output: {"notes": [...], "judge": {"backend","requests","input_tokens","output_tokens","cost_usd"}}
+
+            usage: sloplint-judge [-o full|json] [--register TEXT] [--backend NAME] [command] [args]
+
+            commands:
+              check         scan paths (or stdin) with the judge's rules only [default]
+              compare A B   which of two passages a plain-prose editor keeps (--drift for rewrites)
+              rules         list the judge's rule catalog (add --json)
+              explain ID    print one rule's question, levels, rationale and fixtures
+              status        say whether a run could happen here, and where the key is, without reading it
+              key set       store the backend's key in the OS keychain (the keychain tool prompts for it)
+              key unset     remove it from the OS keychain
+              version       print the sloplint-judge version
+
+            global options:
+          BANNER
+          o.on("-o", "--output-format FORMAT", %w[full json]) { |v| opts[:format] = v }
+          o.on("--register TEXT", "Who the reader is (default: #{Engine::DEFAULT_REGISTER}).") { |v| opts[:register] = v }
+          o.on("--backend NAME", "Which adapter to use (default: $SLOPLINT_JUDGE_BACKEND or jev).") { |v| opts[:backend] = v }
+          o.on("-h", "--help", "Show this help, including the agent recipe above.") { out.puts(o.help); opts[:help_shown] = true }
+          o.on("-v", "--version", "Print the sloplint-judge version.") { out.puts(VERSION); opts[:version_shown] = true }
+        end
+      end
+    end
+  end
+end

data/lib/sloplint/judge/compare.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative "backend"
+
+module Sloplint
+  module Judge
+    # The pairwise judge: which of two passages a plain-prose editor keeps,
+    # and optionally whether B changed what A says. See docs/JUDGE.md "compare".
+    module Compare
+      KEEP = "A careful editor who wants prose that is plain, specific and economical, and who distrusts polish " \
+             "for its own sake, must run exactly one of A and B %{place}. Which does the editor run?"
+      DRIFT = "Does B state any fact, claim or qualification that A does not, or drop any that A states? " \
+              "Answer yes if the meaning differs in any way a careful reader would notice, no if only the wording differs."
+
+      # The acceptance rule for a rewrite (keep B, drift at or under 0.5, the
+      # lower confidence at or above 0.7) belongs to the phase-two loop; this
+      # reports the inputs and applies nothing. See docs/JUDGE.md "compare".
+      Verdict = Data.define(:keep, :p_keep_b, :keep_confidence, :drift, :drift_confidence, :usage)
+
+      module_function
+
+      # a, b: the passages. place: the register as a venue. drift: also ask
+      # whether B changes what A says. The order sent is randomised because
+      # the judge has a position bias; the answer is mapped back.
+      def run(a, b, place:, backend: Backend.load, drift: false, swap: rand < 0.5)
+        first, second = swap ? [b, a] : [a, b]
+        questions = {
+          "keep" => { "type" => "choice", "instructions" => format(KEEP, place: place),
+                      "criteria" => { "A" => "The editor runs passage A.", "B" => "The editor runs passage B." } }
+        }
+        questions["drift"] = { "type" => "noul", "instructions" => DRIFT } if drift
+        answers = backend.ask({ "A" => first, "B" => second }, questions)
+        keep = answers["keep"] or raise BackendError, "the backend answered no keep question"
+        p_b = probability(keep, swap ? "A" : "B")
+        d = answers["drift"]
+        Verdict.new(keep: p_b >= 0.5 ? "B" : "A", p_keep_b: p_b, keep_confidence: keep.confidence,
+                    drift: d&.probabilities, drift_confidence: d&.confidence, usage: keep.usage)
+      end
+
+      # The keep answer is a choice between the options named A and B. An
+      # answer keyed any other way is a malformed body, which ends `compare`
+      # with exit 3 like any other backend failure, not with a backtrace.
+      def probability(keep, option)
+        probs = keep.probabilities
+        raise BackendError, "the keep answer is #{probs.class}, not a probability for A and one for B" unless probs.is_a?(Hash)
+
+        probs.fetch(option) { raise BackendError, "the keep answer has no probability for #{option}" }
+      end
+    end
+  end
+end