mesa_test 1.2.1 → 2.0.0

data/lib/mesa_test.rb

@@ -8,11 +8,33 @@ require 'net/https'
 require 'thor'
 require 'json'
 require 'base64'
+require 'digest'
 
 MesaDirError = Class.new(StandardError)
 TestCaseDirError = Class.new(StandardError)
 InvalidDataType = Class.new(StandardError)
 GitHubError = Class.new(StandardError)
+TestHubError = Class.new(StandardError)
+
+# Output conventions (CLAUDE.md, "Output"): a bold step heading after one
+# blank line for each phase; green when MESA work succeeded (a build, a
+# passing test, a standalone submission), red when something failed, yellow
+# for a warning (the run continues, but you may want to act), blue only for
+# setup-wizard questions; everything else plain. No color for plumbing.
+def say_step(shell, text)
+  shell.say ''
+  shell.say text, :bold
+end
+
+def short_sha(sha)
+  sha.to_s[0, 7]
+end
+
+# A warning on STDERR (so `request_work`'s JSON stays clean), yellow on a
+# terminal.
+def warn_yellow(text)
+  $stderr.puts($stderr.tty? ? "\e[33m#{text}\e[0m" : text)
+end
 
 GITHUB_HTTPS = 'https://github.com/MESAHub/mesa.git'.freeze
 GITHUB_SSH = 'git@github.com:MESAHub/mesa.git'.freeze
@@ -20,86 +42,122 @@ GITHUB_SSH = 'git@github.com:MESAHub/mesa.git'.freeze
 class MesaTestSubmitter
   DEFAULT_URI = 'https://testhub.mesastar.org'.freeze
 
+  # How to get an API key, for the setup wizard and for every command that
+  # finds none.
+  API_KEY_HELP = <<~HELP.freeze
+    mesa_test authenticates with a per-computer API key. To get one:
+      1. Log in to MESATestHub (#{DEFAULT_URI}) and open My Computers.
+      2. Open this computer's page (add the computer first if it isn't
+         listed; a computer used only for `search` and `count` is fine).
+      3. Click "Generate key" and copy the key, which starts with mth_.
+  HELP
+
   # set up config file for computer
   def setup
     update do |s|
-      shell.say 'This wizard will guide you through setting up a computer
-profile and default data for test case submissions to MESATestHub. You
-will be able to confirm entries at the end. Default/current values are always
-shown in parentheses at the end of a prompt. Pressing enter will accept the
-default values.
-
-To submit to MESATestHub, a valid computer name, email address, and password
-are all required. To actually run a test, you need to specify a location for
-your base MESA git repository. All other data are useful, but optional. Any data
-transferred to MESATestHub will be encrypted via HTTPS, but be warned that your
-e-mail and password will be stored in plain text.'
-      # Get computer name
-      response = shell.ask('What is the name of this computer (required)? ' \
-        "(#{s.computer_name}):", :blue)
-      s.computer_name = response unless response.empty?
-
-      # Get user e-mail
-      response = shell.ask 'What is the email you can be reached ' \
-        "at (required)? (#{s.email}):", :blue
-      s.email = response unless response.empty?
-
-      # Get user password
-      response = shell.ask 'What is the password associated with the email ' \
-        "#{s.email} (required)? (#{s.password})", :blue
-      s.password = response unless response.empty?
+      shell.say <<~INTRO
+        This wizard sets up this computer's profile for MESATestHub. Each
+        question shows the current value; press enter to keep it. You can
+        review everything at the end.
+
+        #{API_KEY_HELP}
+        (MESATESTHUB_API_KEY, if set, overrides the key saved here.) The key
+        can only act as this one computer, can't log in to the website, and
+        can be revoked on its own, but it is stored in plain text in
+        #{config_file}, as is the logs token.
+      INTRO
+
+      ask_for_api_key(s)
+
+      # Optional run modes this computer can serve when MESATestHub asks for
+      # them (see `mesa_test request_work` and `install_and_test best`)
+      current = s.capabilities.select { |_, on| on }.keys.join(', ')
+      response = question('Which optional test modes can this computer run when a ' \
+                          'commit asks for them? Any of: full_inlists, fpe, converge, ' \
+                          "separated by commas, or 'none'.", current.empty? ? 'none' : current)
+      s.capabilities = parse_capabilities(response) unless response.empty?
 
       # Get API key for submitting failure logs
-      response = shell.ask 'What is the logs submission API token associated '\
-        "with the email #{s.email} (required; if you don't have one, ask a "\
-        "MESA testing maintainer or a representative from Flatiron for a "\
-        "key)? (#{s.logs_token})", :blue
+      response = question('What is your token for uploading failure logs to the logs ' \
+                          "server? (If you don't have one, ask a MESA testing maintainer " \
+                          'or a representative from Flatiron for a key.)', s.logs_token)
       s.logs_token = response unless response.empty?
 
       # Determine if we'll use ssh or https to access github
-      response = shell.ask 'When accessing GitHub, which protocol do you '\
-        'want to use?', :blue, limited_to: %w[ssh https]
+      shell.say ''
+      shell.say 'When accessing GitHub, which protocol do you want to use?', :blue
+      response = shell.ask '>', limited_to: %w[ssh https], default: s.github_protocol.to_s
       s.github_protocol = response.strip.downcase.to_sym
 
       # Get location of source MESA repo (the mirror)
-      response = shell.ask "Where is/should your mirrored MESA repository " \
-      "located? This is where a mirror will be stored from which test " \
-      "repos will be generated. You won't touch this in regular operation. " \
-      "(#{s.mesa_mirror}):", :blue
+      response = question('Where is/should your mirrored MESA repository be? This is ' \
+                          'where a mirror will be stored from which test repos will be ' \
+                          "generated. You won't touch this in regular operation.", s.mesa_mirror)
       s.mesa_mirror = response unless response.empty?
 
       # Get location of source MESA work (where testing happens)
-      response = shell.ask "Where is/should your working directory for "\
-      "testing be located? This is where testing actually occurs, but all "\
-      "files it uses are cached in the mirror repo to save time later. " \
-      "(#{s.mesa_work}):", :blue
+      response = question('Where is/should your working directory for testing be? This ' \
+                          'is where testing actually occurs, but all files it uses are ' \
+                          'cached in the mirror repo to save time later.', s.mesa_work)
       s.mesa_work = response unless response.empty?
-
-      # Get platform information
-      response = shell.ask 'What is the platform of this computer (eg. ' \
-        "macOS, Ubuntu)? (#{s.platform}):", :blue
-      s.platform = response unless response.empty?
-      response = shell.ask 'What is the version of the platform (eg. 10.15.5, ' \
-        "Ubuntu 16.04)? (#{s.platform_version}):", :blue
-      s.platform_version = response unless response.empty?
-
-      # we are powerless to do change the location for now, so stop asking
-      # about it
-      # # Confirm save location
-      # response = shell.ask "This will be saved in #{s.config_file}. Press " \
-      #   'enter to accept or enter a new location:', :blue, path: true
-      # s.config_file = response unless response.empty?
     end
 
     # Confirm data. If not confirmed, restart whole wizard.
     if confirm_computer_data
       save_computer_data
     else
-      shell.say "Restarting wizard.\n"
+      shell.say 'Starting over.'
       setup
     end
   end
 
+  # Ask one wizard question; returns the stripped answer, '' to keep
+  # +current+. The question goes out through `say`, which the terminal
+  # wraps normally; the prompt that Readline draws stays a short, plain
+  # "> ". Readline counts color codes as visible characters, so a long or
+  # colored prompt garbles at the terminal's edge (the start of the line
+  # repeats before it wraps).
+  def question(text, current = nil)
+    shell.say ''
+    shell.say(current.to_s.empty? ? text : "#{text} (current: #{current})", :blue)
+    shell.ask('>').to_s.strip
+  end
+
+  # Ask for the API key until MESATestHub accepts one, and take the computer
+  # name from it, since a key belongs to exactly one computer. If the hub
+  # can't be reached, keep the key and ask for the name instead.
+  def ask_for_api_key(s)
+    loop do
+      if ENV['MESATESTHUB_API_KEY']
+        shell.say "Using the API key in MESATESTHUB_API_KEY (#{s.masked_api_key}). " \
+                  'Unset it to enter one here.', :yellow
+      else
+        response = question('What is the MESATestHub API key for this computer?',
+                            s.masked_api_key || 'none')
+        s.api_key = response unless response.empty?
+        next shell.say('An API key is required.', :red) unless s.api_key
+      end
+
+      check = s.confirm_computer(computer_name: nil)
+      if check['valid']
+        s.computer_name = check['computer'] || s.computer_name
+        shell.say "Key accepted: this computer is #{s.computer_name} on MESATestHub.", :green
+        return
+      elsif check['reachable'] == false
+        shell.say 'Saving the key unverified.', :yellow
+        response = question('What is the name of this computer on MESATestHub?', s.computer_name)
+        s.computer_name = response unless response.empty?
+        return
+      end
+
+      shell.say check['message'], :red
+      if ENV['MESATESTHUB_API_KEY']
+        raise TestHubError, 'MESATESTHUB_API_KEY was refused; fix or unset it.'
+      end
+      s.api_key = nil
+    end
+  end
+
   def self.new_from_config(
     config_file: File.join(ENV['HOME'], '.mesa_test', 'config.yml'),
     force_setup: false,
@@ -107,6 +165,9 @@ e-mail and password will be stored in plain text.'
   )
     new_submitter = new(config_file: config_file, base_uri: base_uri)
     if force_setup
+      # Start the wizard from the current settings, including a 1.x
+      # config's (logs token, directories), so upgrading keeps them.
+      new_submitter.load_computer_data if File.exist? config_file
       new_submitter.setup
     elsif not File.exist? config_file
       puts "No such config file #{config_file}. Starting setup wizard."
@@ -116,45 +177,41 @@ e-mail and password will be stored in plain text.'
     return new_submitter
   end
 
-  attr_accessor :computer_name, :user_name, :email, :password, :platform,
-                :mesa_mirror, :mesa_work, :platform_version, :processor,
-                :config_file, :base_uri, :github_protocol, :logs_token
+  # Optional run modes a commit can request through its message
+  # (`[ci optional]`, `[ci fpe]`, `[ci converge]`), in MESATestHub's names.
+  CAPABILITIES = %i[full_inlists fpe converge].freeze
+
+  attr_accessor :computer_name, :user_name,
+                :mesa_mirror, :mesa_work, :processor,
+                :config_file, :base_uri, :github_protocol, :logs_token,
+                :capabilities
+  attr_writer :api_key
 
   attr_reader :shell
 
   # many defaults are set in body
   def initialize(
-      computer_name: nil, user_name: nil, email: nil, github_protocol: nil,
-      mesa_mirror: nil, platform: nil, platform_version: nil, processor: nil,
+      computer_name: nil, user_name: nil, github_protocol: nil,
+      mesa_mirror: nil, processor: nil,
       config_file: nil, base_uri: nil, logs_token: nil
   )
     @computer_name = computer_name || Socket.gethostname.scan(/^[^\.]+\.?/)[0]
     @computer_name.chomp!('.') if @computer_name
     @user_name = user_name || (ENV['USER'] || ENV['USERNAME'])
-    @email = email || ''
-    @password = password || ''
     @github_protocol = github_protocol || :ssh
     @mesa_mirror = mesa_mirror ||
       File.join(ENV['HOME'], '.mesa_test', 'mirror')
     @mesa_work = mesa_work ||
       File.join(ENV['HOME'], '.mesa_test', 'work')
-    @platform = platform
-    if @platform.nil?
-      @platform =
-        if OS.osx?
-          'macOS'
-        elsif OS.linux?
-          'Linux'
-        else
-          ''
-        end
-    end
-    @platform_version = platform_version || ''
     @processor = processor || ''
     @config_file = config_file || File.join(ENV['HOME'], '.mesa_test',
                                             'config.yml')
-    @base_uri = base_uri
+    # MESATESTHUB_URL points the client at another testhub (a local dev
+    # server, staging) without editing MODE in bin/mesa_test.
+    @base_uri = ENV['MESATESTHUB_URL'] || base_uri
     @logs_token = logs_token || ENV['MESA_LOGS_TOKEN']
+    @api_key = nil
+    @capabilities = CAPABILITIES.map { |c| [c, false] }.to_h
 
     # set up thor-proof way to get responses from user. Thor hijacks the
     # gets command, so we have to use its built-in "ask" method, which is
@@ -169,71 +226,144 @@ e-mail and password will be stored in plain text.'
     yield self if block_given?
   end
 
+  # The per-computer API key: MESATESTHUB_API_KEY from the environment wins
+  # over the config file, so a cluster job can supply it without the key
+  # ever being written to disk. nil means no key (see #require_api_key!).
+  def api_key
+    key = ENV['MESATESTHUB_API_KEY'] || @api_key
+    key.nil? || key.strip.empty? ? nil : key.strip
+  end
+
+  # Every MESATestHub request needs a key, so commands that talk to the hub
+  # call this first, before checking anything out or building.
+  def require_api_key!
+    return if api_key
+
+    raise TestHubError, <<~MSG
+      No MESATestHub API key is set. (This version of mesa_test no longer
+      signs in with your email and password.)
+
+      #{API_KEY_HELP}
+      Then run `mesa_test setup` and paste it when asked, or export
+      MESATESTHUB_API_KEY, which overrides the config file (e.g. to keep the
+      key off disk on a cluster).
+    MSG
+  end
+
+  def masked_api_key
+    key = api_key
+    key && "#{key[0, 10]}..."
+  end
+
+  # "fpe, full_inlists" => { full_inlists: true, fpe: true, converge: false }
+  def parse_capabilities(text)
+    wanted = text.downcase.split(/[\s,]+/).map(&:to_sym)
+    CAPABILITIES.map { |c| [c, wanted.include?(c)] }.to_h
+  end
+
   def confirm_computer_data
-    puts 'Ready to submit the following data:'
-    puts '-------------------------------------------------------'
-    puts "Computer Name           #{computer_name}"
-    puts "User email              #{email}"
-    puts 'Password                ***********'
-    puts "logs API token          #{logs_token}"
-    puts "GitHub Protocol         #{github_protocol}"
-    puts "MESA Mirror Location    #{mesa_mirror}"
-    puts "MESA Work Location      #{mesa_work}"
-    puts "Platform                #{platform} #{platform_version}"
-    # puts "Config location         #{config_file}"
-    puts '-------------------------------------------------------'
-    puts ''
-    response = shell.ask 'Is this correct? (y/Y = Yes, anything else = No):'
-    response.strip.casecmp('y').zero?
-  end
-
-  # For one "computer" on the web server, and for [subjective] consistency
-  # reasons, the platform, processor, and RAM cannot be changed! If you
-  # change platforms (i.e. switch from mac to linux, or change between linux
-  # flavors), you should create a new computer account. Similarly, create new
-  # computer accounts if you change your RAM or processor. You do not need
-  # to change computers if you upgrade your platform (macOS 10.12 -> 10.13
+    modes = capabilities.select { |_, on| on }.keys.join(', ')
+    say_step(shell, "This computer's settings")
+    [['Computer', computer_name],
+     ['API key', masked_api_key || 'none'],
+     ['Optional test modes', modes.empty? ? 'none' : modes],
+     ['Logs token', logs_token.to_s.empty? ? 'none' : logs_token],
+     ['GitHub protocol', github_protocol],
+     ['MESA mirror', mesa_mirror],
+     ['Work directory', mesa_work],
+     ['OS (detected)', platform_version]].each do |label, value|
+      shell.say format('  %-20s %s', label, value)
+    end
+    shell.say ''
+    shell.ask('Save these settings? [y/N]').to_s.strip.casecmp('y').zero?
+  end
+
   def save_computer_data
     data_hash = {
       'computer_name' => computer_name,
-      'email' => email,
-      'password' => password,
+      'api_key' => @api_key,
+      'capabilities' => capabilities.select { |_, on| on }.keys.map(&:to_s),
       'logs_token' => logs_token,
       'github_protocol' => github_protocol,
       'mesa_mirror' => mesa_mirror,
-      'mesa_work' => mesa_work,
-      'platform' => platform,
-      'platform_version' => platform_version,
+      'mesa_work' => mesa_work
     }
     # make sure there's a directory to write to
     unless dir_or_symlink_exists? File.dirname(config_file)
       FileUtils.mkdir_p File.dirname(config_file)
     end
-    File.open(config_file, 'w') { |f| f.write(YAML.dump(data_hash)) }
+    # Owner-only: the file holds the API key and logs token.
+    File.write(config_file, YAML.dump(data_hash), perm: 0o600)
+    File.chmod(0o600, config_file)
   end
 
+  # A setting missing from the file (e.g. capabilities in a 1.x config)
+  # keeps its default.
   def load_computer_data
-    data_hash = YAML.load(File.read(config_file))
-    @computer_name = data_hash['computer_name']
-    @email = data_hash['email']
-    @password = data_hash['password']
-    @logs_token = data_hash['logs_token']
-    @github_protocol = data_hash['github_protocol'].to_sym
-    @mesa_mirror = data_hash['mesa_mirror']
-    @mesa_work = data_hash['mesa_work']
-    @platform = data_hash['platform']
-    @platform_version = data_hash['platform_version']
+    data_hash = YAML.load(File.read(config_file)) || {}
+    @computer_name = data_hash.fetch('computer_name', @computer_name)
+    @api_key = data_hash.fetch('api_key', @api_key)
+    if data_hash.key?('capabilities')
+      @capabilities = parse_capabilities(Array(data_hash['capabilities']).join(','))
+    end
+    @logs_token = data_hash['logs_token'] || @logs_token
+    @github_protocol = (data_hash['github_protocol'] || @github_protocol).to_sym
+    @mesa_mirror = data_hash.fetch('mesa_mirror', @mesa_mirror)
+    @mesa_work = data_hash.fetch('mesa_work', @mesa_work)
+  end
+
+  # This computer's OS version, detected (once per run) rather than
+  # configured, since a typed-in value goes stale with the first OS update.
+  # MESATestHub shows it after the computer's platform (macOS / linux), so
+  # macOS sends just the number and Linux the distribution:
+  # "27.0.1", "Rocky Linux 8.10", "Ubuntu 24.04 (WSL)".
+  def platform_version
+    @platform_version ||= self.class.detect_platform_version
+  end
+
+  def self.detect_platform_version
+    version = OS.mac? ? `sw_vers -productVersion 2>/dev/null`.strip : linux_version
+    version = `uname -sr 2>/dev/null`.strip if version.to_s.empty?
+    version
+  rescue StandardError
+    `uname -sr 2>/dev/null`.strip rescue ''
+  end
+
+  # The freedesktop os-release standard (every current distribution, and
+  # containers; /usr/lib/os-release is its fallback location):
+  # NAME="Rocky Linux", VERSION_ID="8.10". Rolling releases (Arch) have no
+  # VERSION_ID. RHEL/CentOS 6 predate it but have /etc/redhat-release.
+  def self.linux_version
+    path = %w[/etc/os-release /usr/lib/os-release].find { |f| File.readable?(f) }
+    name =
+      if path
+        os = File.readlines(path, chomp: true)
+                 .map { |line| line.split('=', 2) }.select { |kv| kv.size == 2 }
+                 .to_h { |k, v| [k.strip, v.strip.delete('"\'')] }
+        [os['NAME'], os['VERSION_ID']].compact.join(' ')
+      elsif File.readable?('/etc/redhat-release')
+        # "CentOS release 6.10 (Final)"
+        File.read('/etc/redhat-release').strip.sub(/ \(.*\)\z/, '')
+      end
+    return name if name.to_s.empty?
+
+    wsl = File.readable?('/proc/version') && File.read('/proc/version').match?(/microsoft/i)
+    wsl ? "#{name} (WSL)" : name
   end
 
   # Parameters to be submitted in JSON format for reporting information about
-  # the submitting user and computer
+  # the submitting computer. The API key identifies it; the name is a
+  # cross-check (the hub refuses a key that belongs to another computer).
   def submitter_params
-    {
-      email: email,
-      password: password,
-      computer: computer_name,
-      platform_version: platform_version
-    }
+    { computer: computer_name, platform_version: platform_version }
+  end
+
+  # Headers for every MESATestHub request.
+  def testhub_headers(json_body: true)
+    headers = { 'Accept' => 'application/json' }
+    headers['Content-Type'] = 'application/json' if json_body
+    headers['Authorization'] = "Bearer #{api_key}" if api_key
+    headers
   end
 
   # Parameters to be submitted in JSON format for reporting information about
@@ -261,28 +391,55 @@ e-mail and password will be stored in plain text.'
   # test cases and the test results. These will be encoded as an array of
   # JSON objects.
   def instance_params(mesa)
-    has_errors = []
-    res = []
-    mesa.test_case_names.each do |mod, names|
-      names.each do |test_name|
-        begin
-          test_case = mesa.test_cases[mod][test_name]
-          res << test_case.results_hash
-        rescue TestCaseDirError
-          # shell.say "It appears that #{test_case.test_name} has not been "\
-          #           'run yet. Unable to submit data for this test.', :red
-          has_errors << test_case
+    submittable_tests(mesa).map(&:results_hash)
+  end
+
+  # Tests with results from *this* build: run, readable, and not older than
+  # the build itself. A reused work directory (`install --no-checkout`) can
+  # still hold testhub.yml files from an earlier build; those would be filed
+  # under the wrong commit, so they're skipped (and counted).
+  def submittable_tests(mesa)
+    built_at = mesa.built_at
+    unread = []
+    stale = 0
+    tests = mesa.test_case_names.flat_map do |mod, names|
+      names.filter_map do |test_name|
+        test_case = mesa.test_cases[mod][test_name]
+        next unless test_case&.ran?
+
+        if built_at && File.mtime(test_case.testhub_file) < built_at
+          stale += 1
+          next
         end
+        test_case.results_hash
+        test_case
+      rescue TestCaseDirError, Psych::Exception
+        unread << test_case
+        nil
       end
     end
-    unless has_errors.empty?
-      shell.say "The following test cases could NOT be read for submission:",
-                :red
-      has_errors.each do |test_case|
-        shell.say "- #{test_case.test_name}", :red
-      end
+    if stale.positive?
+      shell.say "Skipping #{stale} test result#{'s' unless stale == 1} older than this " \
+                'build (left over from an earlier one).', :yellow
     end
-    res
+    unless unread.empty?
+      shell.say "Couldn't read the results of #{unread.map(&:test_name).join(', ')}; " \
+                'not submitting them.', :red
+    end
+    tests
+  end
+
+  # The build's recorded run modes in the submissions API's `claim:` block
+  # (use_* flags; informational on the server, which reads the modes each
+  # test actually ran with from its testhub.yml).
+  def run_mode_params(mesa)
+    modes = mesa.recorded_run_modes
+    {
+      use_fpe: modes[:fpe] == true,
+      use_full_inlists: modes[:skip_optional] == false,
+      use_converge: modes[:resolution_factor].is_a?(String) ||
+                    modes[:resolution_factor].is_a?(Numeric)
+    }
   end
 
   # Parameters for reporting a failed compilation to the logs server
@@ -314,28 +471,39 @@ e-mail and password will be stored in plain text.'
     [test_case.results_hash]
   end
 
-  # Phone home to testhub and confirm that computer and user are valid. Useful
-  # for confirming that submissions will be accepted before wasting time on a
-  # test later.
-  def confirm_computer
+  # Phone home to testhub and confirm that the API key is valid (and, given a
+  # computer name, that it belongs to that computer). Useful for confirming
+  # that submissions will be accepted before wasting time on a test later.
+  # 'reachable' is false when the hub couldn't be asked at all.
+  def confirm_computer(computer_name: self.computer_name)
     uri = URI.parse(base_uri + '/check_computer.json')
     https = Net::HTTP.new(uri.hostname, uri.port)
     https.use_ssl = base_uri.include? 'https'
 
-    request = Net::HTTP::Post.new(
-      uri, initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' }
-    )
-    request.body = {
-      email: email,
-      password: password,
-      computer_name: computer_name
-    }.to_json
+    request = Net::HTTP::Post.new(uri, testhub_headers)
+    request.body = { computer_name: computer_name }.compact.to_json
     response = testhub_request(https, request)
     # if the hub was unreachable, behave as an unverified computer; the
     # network error has already been reported by testhub_request
-    return {} if response.nil?
+    if response.nil?
+      return { 'valid' => false, 'reachable' => false,
+               'message' => 'Could not verify this computer with MESATestHub.' }
+    end
+
+    # Anything but a 200 (a 429 from the rate limiter, a 5xx, a proxy's HTML
+    # error page) carries no valid/message keys, so name the status and the
+    # server's error instead of failing with a blank line.
+    unless response.is_a?(Net::HTTPOK)
+      return { 'valid' => false,
+               'message' => 'Could not verify this computer with MESATestHub ' \
+                            "(HTTP #{response.code}: #{error_message(response)})." }
+    end
 
     JSON.parse(response.body).to_hash
+  rescue JSON::ParserError
+    { 'valid' => false,
+      'message' => 'Could not verify this computer: MESATestHub sent an ' \
+                   'unreadable response.' }
   end
 
   # submit entire commit's worth of test cases, OR submit compilation status
@@ -349,10 +517,7 @@ e-mail and password will be stored in plain text.'
     https = Net::HTTP.new(uri.hostname, uri.port)
     https.use_ssl = true if base_uri.include? 'https'
 
-    request = Net::HTTP::Post.new(
-      uri,
-      initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' }
-    )
+    request = Net::HTTP::Post.new(uri, testhub_headers)
 
     # create the request body for submission to the submissions API
     # 
@@ -363,11 +528,20 @@ e-mail and password will be stored in plain text.'
     empty ||= !mesa.installed?
     request_data = {submitter: submitter_params,
                     commit: commit_params(mesa, empty: empty, entire: !empty)}
+    request_data[:claim] = run_mode_params(mesa) if mesa.recorded_run_modes.any?
+    # Fingerprints let MESATestHub ignore results it already has (a retried
+    # submit, a head node re-sending what compute nodes sent).
+    fingerprints = {}
+    fingerprints['build'] = mesa.build_fingerprint if mesa.build_fingerprint
     # don't need test instances if it's an empty submission or if compilation
     # failed
+    tests = []
     if !empty && request_data[:commit][:compiled]
-      request_data[:instances] = instance_params(mesa)
+      tests = submittable_tests(mesa)
+      request_data[:instances] = tests.map(&:results_hash)
+      tests.each { |t| fingerprints[t.fingerprint_key] = t.fingerprint }
     end
+    request_data[:fingerprints] = fingerprints unless fingerprints.empty?
     request.body = request_data.to_json
 
     # actually do the submission
@@ -377,41 +551,39 @@ e-mail and password will be stored in plain text.'
       # network failure; testhub_request already explained why
       false
     elsif !response.is_a? Net::HTTPCreated
-      shell.say "\nFailed to submit some or all test case instances and/or "\
-                "commit data (server responded #{response.code} "\
-                "#{response.message}).", :red
+      shell.say "MESATestHub refused the results for #{short_sha(mesa.sha)} " \
+                "(HTTP #{response.code}: #{error_message(response)}).", :red
       false
     else
-      shell.say "\nSuccessfully submitted commit #{mesa.sha}.", :green
-      # commit submitted to testhub, now submit build log if compilation failed
-      # and exit
+      if !mesa.installed?
+        shell.say "Reported the failed build of #{short_sha(mesa.sha)} to MESATestHub."
+      elsif empty || tests.empty?
+        shell.say "Reported the build of #{short_sha(mesa.sha)} to MESATestHub."
+      else
+        shell.say "Submitted the build and #{tests.size} test result#{'s' unless tests.size == 1} " \
+                  "for #{short_sha(mesa.sha)}.", :green
+      end
+      # Upload the build log if compilation failed. A logs-server problem is
+      # only a warning: the results are already in.
       unless mesa.installed?
-        return submit_build_log(mesa)
+        submit_build_log(mesa)
+        return true
       end
 
-      # compilation succeded, so submit any logs for failing tests
-      res = true
-      unless empty
-        mesa.test_cases.each do |mod, test_case_hash|
-          test_case_hash.each do |tc_name, test_case|
-            # get at each individual test case, see if it failed, and if it
-            # did, submit its log files
-            if !test_case.passed? || force_logs
-              res &&= submit_test_log(test_case, skip_passing: !force_logs)
-            end
-          end
-        end
+      # logs for the failing tests just submitted (or all, with force_logs)
+      tests.each do |test_case|
+        submit_test_log(test_case, skip_passing: !force_logs) if !test_case.passed? || force_logs
       end
-
-      # a true return value means that any and all log submission were
-      # successful
-      res
+      true
     end
   end
 
   # submit results for a single test case instance. Does *not* report overall
-  # compilation status to testhub. Use an empty commit submission for that
-  def submit_instance(mesa, test_case, force_logs: false)
+  # compilation status to testhub. Use an empty commit submission for that.
+  # +quiet+ leaves the success line to the caller (a test run's progress
+  # line says "submitted"). Returns whether the result went in; a log
+  # upload problem is only a warning.
+  def submit_instance(mesa, test_case, force_logs: false, quiet: false)
     unless mesa.install_attempted?
       raise MesaDirError, 'No testhub.yml file found in installation; '\
                           'must attempt to install before subitting.'
@@ -421,10 +593,7 @@ e-mail and password will be stored in plain text.'
     https = Net::HTTP.new(uri.hostname, uri.port)
     https.use_ssl = true if base_uri.include? 'https'
 
-    request = Net::HTTP::Post.new(
-      uri,
-      initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' }
-    )
+    request = Net::HTTP::Post.new(uri, testhub_headers)
 
     # create the request body for submission to the submissions API
     # 
@@ -432,7 +601,9 @@ e-mail and password will be stored in plain text.'
     # entire (... there is only test case)
     request_data = {submitter: submitter_params,
                     commit: commit_params(mesa, empty: false, entire: false),
-                    instances: single_instance_params(test_case)}
+                    instances: single_instance_params(test_case),
+                    fingerprints: { test_case.fingerprint_key => test_case.fingerprint }}
+    request_data[:claim] = run_mode_params(mesa) if mesa.recorded_run_modes.any?
     request.body = request_data.to_json
 
     # actually do the submission
@@ -442,17 +613,13 @@ e-mail and password will be stored in plain text.'
       # network failure; testhub_request already explained why
       return false
     elsif !response.is_a? Net::HTTPCreated
-      shell.say "\nFailed to submit #{test_case.test_name} for commit "\
-                "#{mesa.sha} (server responded #{response.code} "\
-                "#{response.message}).", :red
+      shell.say "MESATestHub refused #{test_case.mod}/#{test_case.test_name} " \
+                "(HTTP #{response.code}: #{error_message(response)}).", :red
       return false
     else
-      shell.say "\nSuccessfully submitted instance of #{test_case.test_name} "\
-                "for commit #{mesa.sha}.", :green
+      shell.say "Submitted #{test_case.mod}/#{test_case.test_name}.", :green unless quiet
       # submit logs if test failed
-      if !test_case.passed? || force_logs
-        return submit_test_log(test_case, skip_passing: !force_logs)
-      end
+      submit_test_log(test_case, skip_passing: !force_logs) if !test_case.passed? || force_logs
       true
     end
   end
@@ -468,8 +635,7 @@ e-mail and password will be stored in plain text.'
     https.read_timeout = 60
     https.request(request)
   rescue StandardError => e
-    shell.say "\nCould not reach the test hub at #{https.address} "\
-              "(#{e.class}: #{e.message}).", :red
+    shell.say "Could not reach MESATestHub at #{https.address} (#{e.class}: #{e.message}).", :red
     nil
   end
 
@@ -496,9 +662,8 @@ e-mail and password will be stored in plain text.'
     begin
       https.request(req)
     rescue StandardError => e
-      shell.say "\nCould not reach the LOGS server at #{uri.host} "\
-                "(#{e.class}: #{e.message}). Skipping log upload; this does "\
-                'not affect test results already sent to the test hub.', :yellow
+      shell.say "Could not reach the logs server at #{uri.host} (#{e.class}: #{e.message}); " \
+                'skipping the log upload. The results themselves are in.', :yellow
       nil
     end
   end
@@ -508,12 +673,7 @@ e-mail and password will be stored in plain text.'
     # intercept and don't send if mesa was properly installed
     return true if mesa.installed?
 
-    # don't even try unless we have a logs token set
-    unless logs_token
-      shell.say 'Cannot submit to logs server; need to set mesa_logs_token '\
-                'in the mesa_test config file.'
-      return false
-    end
+    return false unless logs_token?
 
     # do submission
     res = submit_logs(build_log_params(mesa))
@@ -523,13 +683,10 @@ e-mail and password will be stored in plain text.'
       # network failure; submit_logs already explained why. Don't fail the run.
       false
     elsif !res.is_a? Net::HTTPOK
-      shell.say "\nFailed to submit build.log to the LOGS server for commit "\
-                "#{mesa.sha} (server responded #{res.code} #{res.message}).",
-                :red
+      shell.say "The logs server refused the build log (HTTP #{res.code} #{res.message}).", :yellow
       false
     else
-      shell.say "\nSuccessfully submitted build.log to the LOGS server for "\
-                "#{mesa.sha}.", :green
+      shell.say 'Uploaded the build log to the logs server.'
       true
     end
   end
@@ -542,13 +699,8 @@ e-mail and password will be stored in plain text.'
       return true
     end
 
-    # don't even try unless we have a logs token set
-    unless logs_token
-      shell.say 'Cannot submit to logs server; need to set mesa_logs_token '\
-                'in the mesa_test config file..'
-      return false
-    end
-  
+    return false unless logs_token?
+
     # do submission
     res = submit_logs(test_log_params(test_case))
 
@@ -557,21 +709,29 @@ e-mail and password will be stored in plain text.'
       # network failure; submit_logs already explained why. Don't fail the run.
       false
     elsif !res.is_a? Net::HTTPOK
-      shell.say "Failed to submit logs for test case #{test_case.test_name} "\
-                "in commit #{test_case.mesa.sha} (server responded "\
-                "#{res.code} #{res.message}).", :red
+      shell.say "The logs server refused the logs for #{test_case.test_name} " \
+                "(HTTP #{res.code} #{res.message}).", :yellow
       false
     else
-      shell.say "Successfully submitted logs for test case "\
-                "#{test_case.test_name} in commit #{test_case.mesa.sha}.",
-                :green
+      shell.say "Uploaded the logs for #{test_case.test_name} to the logs server."
       true
     end
   end
 
-  # GET a search query from the testhub. Credentials and the query string
-  # ride in the URL's query parameters (per the testhub search API contract;
-  # the controller also accepts session auth, but a CLI sends them in-band).
+  # Whether failure logs can be uploaded; says why not once per run.
+  def logs_token?
+    return true if logs_token && !logs_token.to_s.empty?
+
+    unless @warned_no_logs_token
+      shell.say 'No logs token is set, so failure logs are not being uploaded ' \
+                '(add one with `mesa_test setup`).', :yellow
+      @warned_no_logs_token = true
+    end
+    false
+  end
+
+  # GET a search query from the testhub. The query rides in the URL's
+  # query_text parameter; the API key in the Authorization header.
   # Returns the raw Net::HTTP response.
   def search(query_text)
     get_search('/test_instances/search.json', query_text)
@@ -583,23 +743,112 @@ e-mail and password will be stored in plain text.'
     get_search('/test_instances/search_count.json', query_text)
   end
 
+  # What a claimed run is running on, for MESATestHub to show on pending
+  # tests (so a long-running claim can be explained: an old machine on two
+  # threads, a different SDK). Only what's actually known is sent:
+  #   - with a built +mesa+, its testhub.yml compiler record (SDK, compiler,
+  #     math backend), which is accurate even if the shell's SDK changed
+  #   - otherwise (a build claim, made before compiling) just the SDK the
+  #     shell has active, from mesasdk_init.sh's MESASDK_VERSION
+  #   - OMP_NUM_THREADS and this computer's platform version
+  def claim_environment(mesa = nil)
+    env = {}
+    if mesa&.install_attempted?
+      env.merge!(mesa.compiler_hash.reject { |_k, v| v.to_s.strip.empty? || v == 'Unknown' })
+    end
+    sdk = ENV['MESASDK_VERSION'].to_s.strip
+    env[:sdk_version] ||= sdk unless sdk.empty?
+    threads = ENV['OMP_NUM_THREADS'].to_i
+    env[:omp_num_threads] = threads if threads.positive?
+    env[:platform_version] = platform_version unless platform_version.to_s.strip.empty?
+    env
+  end
+
+  # Ask MESATestHub what this computer should do next (POST /api/v1/dispatch).
+  # +scope+ is 'build' (which commit to build) or 'test' (which test to run;
+  # pass the +sha+ you built). +capabilities+ defaults to the configured
+  # optional modes; for a test, +fpe+ should say whether *this build* has
+  # FPE checks, since MESA_FPE_CHECKS_ON is a compile-time switch.
+  #
+  # Returns the recommendation as a Hash (commit_sha, branch, scope,
+  # test_case_module, test_case_name, flags, reasons, ...), or nil when
+  # there's nothing useful to do. Raises TestHubError on any other answer.
+  def dispatch(scope:, sha: nil, capabilities: self.capabilities)
+    body = { scope: scope }
+    body[:commit_sha] = sha if sha
+    CAPABILITIES.each { |c| body[:"can_#{c}"] = !!capabilities[c] }
+    response = post_testhub('/api/v1/dispatch', submitter: submitter_params, dispatch: body)
+    raise TestHubError, 'Dispatch failed: could not reach MESATestHub.' if response.nil?
+    return nil if response.is_a?(Net::HTTPNoContent)
+    return JSON.parse(response.body) if response.is_a?(Net::HTTPSuccess)
+
+    raise TestHubError, "Dispatch failed (HTTP #{response.code}): #{error_message(response)}"
+  end
+
+  # Tell MESATestHub this computer is about to build +sha+ (scope 'build') or
+  # run a test on it (scope 'test' with +mod+ and +test_case+, or
+  # +all_test_cases: true+ for the whole suite). The commit then shows as
+  # pending instead of untested. The later submission fulfills the claim by
+  # matching computer, commit, and test, so nothing needs remembering here.
+  #
+  # Claims are advisory: a failure is reported and swallowed (returns nil) so
+  # it never stops a test run.
+  #
+  # The claim also says what the run is running on (see #claim_environment),
+  # which MESATestHub shows on the pending test. Pass +mesa+ for test claims
+  # so the build's own compiler record is used.
+  def claim(sha:, scope:, mod: nil, test_case: nil, all_test_cases: false,
+            flags: {}, dispatched_at: nil, mesa: nil)
+    body = { commit_sha: sha, scope: scope }
+    body.merge!(claim_environment(scope.to_s == 'test' ? mesa : nil))
+    body[:test_case_module] = mod.to_s if mod
+    body[:test_case_name] = test_case if test_case
+    body[:all_test_cases] = true if all_test_cases
+    body[:dispatched_at] = dispatched_at if dispatched_at
+    CAPABILITIES.each { |c| body[:"use_#{c}"] = !!flags[c] }
+    response = post_testhub('/api/v1/claims', submitter: submitter_params, claim: body)
+    return nil if response.nil? # unreachable; testhub_request said so
+    return JSON.parse(response.body) if response.is_a?(Net::HTTPCreated)
+
+    shell.say "Could not claim #{claim_label(scope, mod, test_case, all_test_cases)} " \
+              "(HTTP #{response.code}: #{error_message(response)}); continuing anyway.",
+              :yellow
+    nil
+  end
+
   private
 
+  # POST JSON to the hub with testhub_request's bounded timeouts. Returns
+  # the response, or nil after a network failure (already reported).
+  def post_testhub(path, payload)
+    uri = URI.parse(base_uri + path)
+    https = Net::HTTP.new(uri.hostname, uri.port)
+    https.use_ssl = base_uri.include? 'https'
+    request = Net::HTTP::Post.new(uri, testhub_headers)
+    request.body = payload.to_json
+    testhub_request(https, request)
+  end
+
+  def error_message(response)
+    JSON.parse(response.body.to_s)['error'] || response.message
+  rescue JSON::ParserError
+    response.message
+  end
+
+  def claim_label(scope, mod, test_case, all_test_cases)
+    return 'the build' if scope.to_s == 'build'
+    return 'every test' if all_test_cases
+    "#{mod}/#{test_case}"
+  end
+
   def get_search(path, query_text)
     uri = URI.parse(base_uri + path)
-    uri.query = URI.encode_www_form(
-      email: email,
-      password: password,
-      query_text: query_text
-    )
+    uri.query = URI.encode_www_form(query_text: query_text)
 
     https = Net::HTTP.new(uri.hostname, uri.port)
     https.use_ssl = base_uri.include? 'https'
 
-    request = Net::HTTP::Get.new(
-      uri,
-      initheader = { 'Accept' => 'application/json' }
-    )
+    request = Net::HTTP::Get.new(uri, testhub_headers(json_body: false))
 
     https.request(request)
   end
@@ -643,35 +892,29 @@ class Mesa
   end
 
   def checkout(new_sha: 'HEAD')
+    # A branch name is worth showing; a SHA is shown with its subject below.
+    say_step(shell, new_sha =~ /\A\h{40}\z/ ? 'Checking out MESA' : "Checking out #{new_sha}")
     # before anything confirm that git-lfs has been installed
-    shell.say "\nEnsuring that git-lfs is installed... ", :blue
     command = 'git lfs help >> /dev/null 2>&1'
-    if bash_execute(command)
-      shell.say "yes", :green
-    else
-      shell.say "no", :red
-      raise(GitHubError, "The command #{command} returned with an error "\
-                         'status, indicating that git-lfs is not installed. '\
-                         'Make sure it is installed and try again.')
+    unless bash_execute(command)
+      raise(GitHubError, 'git-lfs is not installed (`git lfs help` failed); ' \
+                         'MESA needs it. Install it and try again.')
     end
 
     # set up mirror if it doesn't exist
     unless dir_or_symlink_exists?(mirror_dir)
-      shell.say "\nCreating initial mirror at #{mirror_dir}. "\
-                'This might take awhile...', :blue
+      shell.say "Creating a mirror of MESA in #{mirror_dir} (first time only; " \
+                'this can take a while).'
       FileUtils.mkdir_p mirror_dir
       case github_protocol
       when :ssh
         command = "git clone --mirror #{GITHUB_SSH} #{mirror_dir}"
-        shell.say command
         # fail loudly if this doesn't work
         unless bash_execute(command)
           # nuke the mirror directory since it is probably bogus (make this
           # code fire off the next time checkout is done)
-          shell.say "Failed. Removing the [potentially corrupted] mirror.", :red
-          command = "rm -rf #{mirror_dir}"
-          shell.say command
-          bash_execute(command)
+          shell.say 'Cloning failed; removing the partial mirror.', :red
+          bash_execute("rm -rf #{mirror_dir}")
 
           raise(GitHubError, 'Error while executing the following command:'\
                              "#{command}. Perhaps you haven't set up "\
@@ -679,15 +922,12 @@ class Mesa
         end
       when :https
         command = "git clone --mirror #{GITHUB_HTTPS} #{mirror_dir}"
-        shell.say command
         # fail loudly if this doesn't work
         unless bash_execute(command)
           # nuke the mirror directory since it is probably bogus (make this
           # code fire off the next time checkout is done)
-          shell.say "Failed. Removing the [potentially corrupted] mirror.", :red
-          command = "rm -rf #{mirror_dir}"
-          shell.say command
-          bash_execute(command)
+          shell.say 'Cloning failed; removing the partial mirror.', :red
+          bash_execute("rm -rf #{mirror_dir}")
 
           raise(GitHubError, 'Error while executing the following command: '\
                              "#{command}. Perhaps you need to configure "\
@@ -705,20 +945,19 @@ class Mesa
     update_mirror
 
     # create "work" directory with proper commit
-    shell.say "\nSetting up worktree repo...", :blue
     FileUtils.mkdir_p mesa_dir
-    command = "git -C #{mirror_dir} worktree add #{mesa_dir} #{new_sha}"
-    shell.say command
-    return if bash_execute(command)
+    command = "git -C #{mirror_dir} worktree add --quiet #{mesa_dir} #{new_sha}"
+    if bash_execute(command)
+      shell.say "#{bashticks("git -C #{mesa_dir} log -1 --format='%h %s'")} (in #{mesa_dir})"
+      return
+    end
 
     raise(GitHubError, 'Failed while executing the following command: '\
                        "\"#{command}\".")
   end
 
   def update_mirror
-    shell.say "\nFetching MESA history...", :blue
-    command = "git -C #{mirror_dir} fetch origin --prune"
-    shell.say command
+    command = "git -C #{mirror_dir} fetch origin --prune --quiet"
     # fail loudly
     return if bash_execute(command)
 
@@ -728,15 +967,11 @@ class Mesa
 
   def remove
     return unless File.exist? mesa_dir
-    shell.say "\nRemoving work directory from worktree (clearing old data)...",
-              :blue
     command = "git -C #{mirror_dir} worktree remove --force #{mesa_dir}"
-    shell.say command
     return if bash_execute(command)
 
-    shell.say "Failed. Simply trying to remove the directory.", :red
+    # Not a worktree of this mirror (or git refused): just delete it.
     command = "rm -rf #{mesa_dir}"
-    shell.say command
     # fail loudly (the "true" tells bash_execute to raise an exception if
     # the command fails)
     bash_execute(command, true)
@@ -754,8 +989,6 @@ class Mesa
     with_mesa_dir do
       visit_and_check mesa_dir, MesaDirError, 'E\countered a problem in ' \
                                 "running `clean` in #{mesa_dir}." do
-        shell.say('MESA_DIR = ' + ENV['MESA_DIR'])
-        shell.say './clean'
         bash_execute('./clean')
       end
     end
@@ -766,8 +999,6 @@ class Mesa
     with_mesa_dir do
       visit_and_check mesa_dir, MesaDirError, 'Encountered a problem in ' \
                                 "running `install` in #{mesa_dir}." do
-        shell.say('MESA_DIR = ' + ENV['MESA_DIR'])
-        shell.say './install'
         bash_execute('./install')
       end
     end
@@ -792,23 +1023,65 @@ class Mesa
     b64_file(build_log)
   end
 
+  COMPILER_KEYS = %i[compiler compiler_version sdk_version math_backend].freeze
+
   # sourced from $MESA_DIR/testhub.yml, which should be created after
-  # installation
+  # installation. Only the compiler keys are taken: the file also carries
+  # mesa_test's own run-mode block (see #record_run_modes), which doesn't
+  # belong in the commit payload.
   def compiler_hash
-    data_file = File.join(mesa_dir, 'testhub.yml')
     res = {
             compiler: 'Unknown',
             sdk_version: 'Unknown',
             math_backend: 'Unknown'
           }
-    if File.exist? data_file
-      res = res.merge(YAML.safe_load(File.read(data_file)) || {})
-      # currently version_number is reported, but we don't need that in Git land
-      res.delete('version_number') # returns the value, not the updated hash
-      res
+    # testhub.yml has string keys; symbolize them so they replace the
+    # defaults above rather than sitting beside them as duplicate keys
+    # (which json >= 3 refuses to serialize).
+    found = install_data.transform_keys(&:to_sym).slice(*COMPILER_KEYS)
+    res.merge(found)
+  end
+
+  # The key under which `mesa_test install` records, in $MESA_DIR/testhub.yml,
+  # the run modes this build was made for. The file is written by MESA's own
+  # install (math/test), so it is recreated on every build and can't go stale.
+  RUN_MODES_KEY = 'mesa_test_run_modes'.freeze
+
+  # Run modes recorded at install, as a hash with any of
+  #   fpe:               true/false (compiled with MESA_FPE_CHECKS_ON?)
+  #   skip_optional:     true/false (MESA_SKIP_OPTIONAL set or unset)
+  #   resolution_factor: a factor string, or false for "unset"
+  # Only modes that were explicitly chosen at install are present; anything
+  # absent falls back to the caller's environment.
+  def recorded_run_modes
+    (install_data[RUN_MODES_KEY] || {}).transform_keys(&:to_sym)
+                                       .slice(*RUN_MODE_KEYS)
+  end
+
+  # Append the modes chosen for this build to testhub.yml. Call after the
+  # install, which rewrites the file.
+  def record_run_modes(modes)
+    data_file = File.join(mesa_dir, 'testhub.yml')
+    return unless File.exist?(data_file)
+
+    modes = modes.slice(*RUN_MODE_KEYS)
+    return if modes.empty?
+
+    block = { RUN_MODES_KEY => modes.transform_keys(&:to_s) }.to_yaml.sub(/\A---\n/, '')
+    File.open(data_file, 'a') do |f|
+      f.puts '# Added by mesa_test install: the run modes this build was made for.'
+      f.puts '# `mesa_test test` applies them unless told otherwise.'
+      f.write block
     end
   end
 
+  def install_data
+    data_file = File.join(mesa_dir, 'testhub.yml')
+    return {} unless File.exist?(data_file)
+
+    YAML.safe_load(File.read(data_file)) || {}
+  end
+
   ## TEST SUITE METHODS
 
   def check_mod(mod)
@@ -839,7 +1112,7 @@ class Mesa
       @names_to_numbers[mod] = {}
       @test_case_names[mod] = []
       @test_cases[mod] = {}
-      visit_dir(test_suite_dir(mod: mod), quiet: true) do
+      visit_dir(test_suite_dir(mod: mod)) do
         bashticks('./list_tests').split("\n").each do |line|
           num, tc_name = line.strip.split
           @names_to_numbers[mod][tc_name.strip] = num.to_i
@@ -852,7 +1125,7 @@ class Mesa
               mesa: self
             )
           rescue TestCaseDirError
-            shell.say "No such test case #{tc_name.strip}. Skipping loading it.", :red
+            shell.say "do1_test_source lists #{tc_name.strip}, but it has no directory; skipping it.", :yellow
           end
         end
       end
@@ -899,6 +1172,18 @@ class Mesa
     )
   end
 
+  # Fingerprint of this build (a hash of build.log), so MESATestHub can
+  # recognize a build report it already has. nil before any build.
+  def build_fingerprint
+    file_fingerprint(File.join(mesa_dir, 'build.log'))
+  end
+
+  # When this build finished (build.log's mtime), or nil.
+  def built_at
+    build_log = File.join(mesa_dir, 'build.log')
+    File.exist?(build_log) ? File.mtime(build_log) : nil
+  end
+
   def install_attempted?
     File.exist? File.join(mesa_dir, 'testhub.yml')
   end
@@ -909,15 +1194,13 @@ class Mesa
     # change MESA_DIR, holding on to old value
     orig_mesa_dir = ENV['MESA_DIR']
     ENV['MESA_DIR'] = mesa_dir
-    shell.say "Temporarily changed MESA_DIR to #{ENV['MESA_DIR']}.", :blue
-  
+
     # do the stuff
     begin
       yield
     # make sure we undo MESA_DIR change
     ensure
       ENV['MESA_DIR'] = orig_mesa_dir
-      shell.say "Changed MESA_DIR back to #{ENV['MESA_DIR']}.", :blue
     end
   end
 
@@ -1073,10 +1356,26 @@ class MesaTestCase
     File.join(test_case_dir, 'testhub.yml')
   end
 
+  # Run this test like #do_one, but as its own subprocess with its own
+  # working directory and environment (+env+ overrides; nil unsets), so
+  # several can run at once -- #do_one changes the process-wide directory
+  # and expects the environment to be set globally. Output goes to
+  # each_test_run.log in the test's directory. Returns true if
+  # each_test_run exited cleanly (the test's pass/fail is in testhub.yml).
+  def run_isolated(env: {})
+    log = File.join(test_case_dir, 'each_test_run.log')
+    pid = Process.spawn(env, './each_test_run', position.to_s,
+                        chdir: test_suite_dir, out: log, err: %i[child out], pgroup: true)
+    RUNNING_TESTS_LOCK.synchronize { RUNNING_TESTS[pid] = test_name }
+    _, status = Process.wait2(pid)
+    status.success?
+  ensure
+    RUNNING_TESTS_LOCK.synchronize { RUNNING_TESTS.delete(pid) } if pid
+  end
+
   # just punt to +each_test_run+ in the test_suite directory. It's your problem
   # now, sucker!
   def do_one
-    shell.say("Testing #{test_name}", :yellow)
     visit_dir(test_suite_dir) do
       bash_execute("./each_test_run #{position}")
     end
@@ -1090,6 +1389,23 @@ class MesaTestCase
     YAML.load(File.read(testhub_file))
   end
 
+  # How MESATestHub keys this test's fingerprint: "<module>/<test>".
+  def fingerprint_key
+    "#{mod}/#{test_name}"
+  end
+
+  # Fingerprint of this run's results (a hash of testhub.yml).
+  def fingerprint
+    file_fingerprint(testhub_file)
+  end
+
+  # Whether this test wrote results at or after +time+, i.e. during the
+  # run that started then (not a testhub.yml left from an earlier run).
+  # mtimes can be whole seconds, so compare at that resolution.
+  def ran_since?(time)
+    ran? && File.mtime(testhub_file) >= time.floor
+  end
+
   # rough proxy for whether or not the test has even been run
   def ran?
     File.exist?(testhub_file)
@@ -1157,12 +1473,8 @@ end
 # exception if the block failed (returned false or nil)
 def visit_and_check(new_dir, exception, message)
   cwd = Dir.getwd
-  shell.say "Leaving  #{cwd}", :blue
-  shell.say "\nEntering #{new_dir}.", :blue
   Dir.chdir(new_dir)
   success = yield if block_given?
-  shell.say "Leaving  #{new_dir}", :blue
-  shell.say "\nEntering #{cwd}.", :blue
   Dir.chdir(cwd)
   return if success
   raise exception, message
@@ -1170,19 +1482,15 @@ end
 
 # cd into a new directory, execute a block, then cd back into original
 # directory
-def visit_dir(new_dir, quiet: false)
+def visit_dir(new_dir)
   cwd = Dir.getwd
-  shell.say "Leaving  #{cwd}\n", :blue unless quiet
-  shell.say "\nEntering #{new_dir}.", :blue unless quiet
   Dir.chdir(new_dir)
   yield if block_given?
-  shell.say "Leaving  #{new_dir}\n", :blue unless quiet
-  shell.say "\nRe-entering #{cwd}.", :blue unless quiet
   Dir.chdir(cwd)
 end
 
 # the next function probalby doesn't belong here, but keep it anyway, please
-# create seed data for test cases for MesaTestHub of a given mesa version
+# create seed data for test cases for MESATestHub of a given mesa version
 def generate_seeds_rb(mesa_dir, outfile)
   m = Mesa.new(mesa_dir: mesa_dir)
   m.load_test_source_data
@@ -1224,6 +1532,137 @@ def bashticks(command)
   `bash -c "#{command}"`.chomp
 end
 
+# Run modes: how MESA is told to run a test, and the environment variables
+# behind them. MESA's each_test_run records each one in the test's
+# testhub.yml (fpe_checks / run_optional / resolution_factor), which is what
+# MESATestHub reads.
+#
+#   fpe:               MESA_FPE_CHECKS_ON=1. Compile-time as well as run-time
+#                      (MESA's make reads it), so it belongs to the build.
+#   skip_optional:     MESA_SKIP_OPTIONAL set. MESA runs every inlist unless
+#                      this is set.
+#   resolution_factor: MESA_TEST_SUITE_RESOLUTION_FACTOR, which is what
+#                      `[ci converge]` sets.
+#
+# A mode that is nil/absent leaves the variable as the user's shell has it.
+RUN_MODE_KEYS = %i[fpe skip_optional resolution_factor].freeze
+DEFAULT_CONVERGE_FACTOR = '0.8'.freeze
+
+# ENV overrides for +modes+ (nil value = unset the variable).
+def run_mode_env(modes)
+  env = {}
+  env['MESA_FPE_CHECKS_ON'] = modes[:fpe] ? '1' : nil unless modes[:fpe].nil?
+  unless modes[:skip_optional].nil?
+    env['MESA_SKIP_OPTIONAL'] = modes[:skip_optional] ? 't' : nil
+  end
+  unless modes[:resolution_factor].nil?
+    env['MESA_TEST_SUITE_RESOLUTION_FACTOR'] =
+      modes[:resolution_factor] ? modes[:resolution_factor].to_s : nil
+  end
+  env
+end
+
+# The resolution factor a converge run should use: the user's own, if their
+# shell sets one, else DEFAULT_CONVERGE_FACTOR.
+def converge_factor
+  ENV['MESA_TEST_SUITE_RESOLUTION_FACTOR'] || DEFAULT_CONVERGE_FACTOR
+end
+
+# Run modes a MESATestHub dispatch asked for. Requests only ever *add* to the
+# user's setup: a mode the hub didn't ask for is left out (so the shell
+# decides), never forced off.
+#
+# A converge request may name its factor (`[ci converge 0.5]` arrives as
+# rec['converge_factor']). That's the commit author's request, so it wins
+# over +preferred+ -- a factor from --resolution-factor or the shell's
+# MESA_TEST_SUITE_RESOLUTION_FACTOR -- with a warning when they differ.
+# Without a named factor, +preferred+ (else the default) is used.
+def dispatch_run_modes(rec, preferred: nil, preferred_source: nil)
+  flags = rec['flags'] || {}
+  modes = {}
+  modes[:fpe] = true if flags['use_fpe']
+  modes[:skip_optional] = false if flags['use_full_inlists']
+  if flags['use_converge']
+    requested = rec['converge_factor']
+    preferred ||= ENV['MESA_TEST_SUITE_RESOLUTION_FACTOR']
+    preferred_source ||= 'MESA_TEST_SUITE_RESOLUTION_FACTOR'
+    if requested
+      factor = requested.to_s
+      if preferred && preferred.to_f != requested.to_f
+        warn_yellow "MESATestHub asked for resolution factor #{factor} ([ci converge #{factor}]); " \
+                    "using it instead of your #{preferred_source}=#{preferred}."
+      end
+      modes[:resolution_factor] = factor
+    else
+      modes[:resolution_factor] = preferred || DEFAULT_CONVERGE_FACTOR
+    end
+  end
+  modes
+end
+
+# Process groups of tests started by MesaTestCase#run_isolated, so an
+# interrupted parallel run can stop them all.
+RUNNING_TESTS = {}
+RUNNING_TESTS_LOCK = Mutex.new
+
+def stop_running_tests
+  pids = RUNNING_TESTS_LOCK.synchronize { RUNNING_TESTS.keys }
+  pids.each do |pid|
+    Process.kill('TERM', -pid)
+  rescue Errno::ESRCH, Errno::EPERM
+    nil
+  end
+end
+
+# Run +jobs+ worker threads, each calling the block with its index until
+# the block returns :done. On Ctrl-C, kills the workers first (so none
+# goes on to submit a half-run test), then every running test, then
+# re-raises. Exceptions in a worker propagate when it's joined.
+def run_workers(jobs)
+  workers = []
+  jobs.times do |i|
+    workers << Thread.new do
+      Thread.current.report_on_exception = false
+      loop { break if yield(i) == :done }
+    end
+  end
+  workers.each(&:join)
+rescue Interrupt
+  workers.each(&:kill)
+  stop_running_tests
+  raise
+end
+
+# Hand each of +items+ to the block, +jobs+ at a time.
+def parallel_each(items, jobs)
+  queue = Queue.new
+  items.each { |item| queue << item }
+  run_workers([jobs, items.size].min) do
+    item = begin
+      queue.pop(true)
+    rescue ThreadError
+      next :done
+    end
+    yield item
+    nil
+  end
+end
+
+# Run the block with +overrides+ applied to ENV (nil deletes a variable),
+# restoring the previous values afterwards.
+def with_env(overrides)
+  saved = overrides.keys.map { |k| [k, ENV[k]] }.to_h
+  overrides.each { |k, v| v.nil? ? ENV.delete(k) : ENV[k] = v }
+  yield
+ensure
+  saved&.each { |k, v| v.nil? ? ENV.delete(k) : ENV[k] = v }
+end
+
+# SHA-256 of a file's contents, or nil if it doesn't exist.
+def file_fingerprint(path)
+  File.exist?(path) ? Digest::SHA256.file(path).hexdigest : nil
+end
+
 # encode the contents of a file as base-64
 def b64_file(filename)
   Base64.encode64(File.open(filename).read)