mesa_test 1.2.0 → 2.0.0.rc1

data/lib/mesa_test.rb

@@ -8,11 +8,13 @@ require 'net/https'
 require 'thor'
 require 'json'
 require 'base64'
+require 'digest'
 
 MesaDirError = Class.new(StandardError)
 TestCaseDirError = Class.new(StandardError)
 InvalidDataType = Class.new(StandardError)
 GitHubError = Class.new(StandardError)
+TestHubError = Class.new(StandardError)
 
 GITHUB_HTTPS = 'https://github.com/MESAHub/mesa.git'.freeze
 GITHUB_SSH = 'git@github.com:MESAHub/mesa.git'.freeze
@@ -20,45 +22,53 @@ GITHUB_SSH = 'git@github.com:MESAHub/mesa.git'.freeze
 class MesaTestSubmitter
   DEFAULT_URI = 'https://testhub.mesastar.org'.freeze
 
+  # How to get an API key, for the setup wizard and for every command that
+  # finds none.
+  API_KEY_HELP = <<~HELP.freeze
+    mesa_test authenticates with a per-computer API key. To get one:
+      1. Log in to MESATestHub (#{DEFAULT_URI}) and open My Computers.
+      2. Open this computer's page (add the computer first if it isn't
+         listed; a computer used only for `search` and `count` is fine).
+      3. Click "Generate key" and copy the key, which starts with mth_.
+  HELP
+
   # set up config file for computer
   def setup
     update do |s|
-      shell.say 'This wizard will guide you through setting up a computer
-profile and default data for test case submissions to MESATestHub. You
-will be able to confirm entries at the end. Default/current values are always
-shown in parentheses at the end of a prompt. Pressing enter will accept the
-default values.
-
-To submit to MESATestHub, a valid computer name, email address, and password
-are all required. To actually run a test, you need to specify a location for
-your base MESA git repository. All other data are useful, but optional. Any data
-transferred to MESATestHub will be encrypted via HTTPS, but be warned that your
-e-mail and password will be stored in plain text.'
-      # Get computer name
-      response = shell.ask('What is the name of this computer (required)? ' \
-        "(#{s.computer_name}):", :blue)
-      s.computer_name = response unless response.empty?
-
-      # Get user e-mail
-      response = shell.ask 'What is the email you can be reached ' \
-        "at (required)? (#{s.email}):", :blue
-      s.email = response unless response.empty?
-
-      # Get user password
-      response = shell.ask 'What is the password associated with the email ' \
-        "#{s.email} (required)? (#{s.password})", :blue
-      s.password = response unless response.empty?
+      shell.say <<~INTRO
+        This wizard sets up this computer's profile for MESATestHub. Current
+        values are shown in parentheses; press enter to keep them. You can
+        confirm everything at the end.
+
+        #{API_KEY_HELP}
+        (MESATESTHUB_API_KEY, if set, overrides the key saved here.) The key
+        can only act as this one computer, can't log in to the website, and
+        can be revoked on its own, but it is stored in plain text in
+        #{config_file}, as is the logs token.
+      INTRO
+
+      ask_for_api_key(s)
+
+      # Optional run modes this computer can serve when MESATestHub asks for
+      # them (see `mesa_test request_work` and `install_and_test best`)
+      current = s.capabilities.select { |_, on| on }.keys.join(', ')
+      response = shell.ask 'Which optional test modes can this computer ' \
+        'run when a commit asks for them? Any of: full_inlists, fpe, ' \
+        "converge, separated by commas, or 'none'. " \
+        "(#{current.empty? ? 'none' : current}):", :blue
+      s.capabilities = parse_capabilities(response) unless response.strip.empty?
 
       # Get API key for submitting failure logs
-      response = shell.ask 'What is the logs submission API token associated '\
-        "with the email #{s.email} (required; if you don't have one, ask a "\
-        "MESA testing maintainer or a representative from Flatiron for a "\
-        "key)? (#{s.logs_token})", :blue
+      response = shell.ask 'What is your token for uploading failure logs to ' \
+        "the logs server? (if you don't have one, ask a MESA testing " \
+        'maintainer or a representative from Flatiron for a key) ' \
+        "(#{s.logs_token}):", :blue
       s.logs_token = response unless response.empty?
 
       # Determine if we'll use ssh or https to access github
       response = shell.ask 'When accessing GitHub, which protocol do you '\
-        'want to use?', :blue, limited_to: %w[ssh https]
+        'want to use?', :blue, limited_to: %w[ssh https],
+        default: s.github_protocol.to_s
       s.github_protocol = response.strip.downcase.to_sym
 
       # Get location of source MESA repo (the mirror)
@@ -74,21 +84,6 @@ e-mail and password will be stored in plain text.'
       "files it uses are cached in the mirror repo to save time later. " \
       "(#{s.mesa_work}):", :blue
       s.mesa_work = response unless response.empty?
-
-      # Get platform information
-      response = shell.ask 'What is the platform of this computer (eg. ' \
-        "macOS, Ubuntu)? (#{s.platform}):", :blue
-      s.platform = response unless response.empty?
-      response = shell.ask 'What is the version of the platform (eg. 10.15.5, ' \
-        "Ubuntu 16.04)? (#{s.platform_version}):", :blue
-      s.platform_version = response unless response.empty?
-
-      # we are powerless to do change the location for now, so stop asking
-      # about it
-      # # Confirm save location
-      # response = shell.ask "This will be saved in #{s.config_file}. Press " \
-      #   'enter to accept or enter a new location:', :blue, path: true
-      # s.config_file = response unless response.empty?
     end
 
     # Confirm data. If not confirmed, restart whole wizard.
@@ -100,6 +95,42 @@ e-mail and password will be stored in plain text.'
     end
   end
 
+  # Ask for the API key until MESATestHub accepts one, and take the computer
+  # name from it, since a key belongs to exactly one computer. If the hub
+  # can't be reached, keep the key and ask for the name instead.
+  def ask_for_api_key(s)
+    loop do
+      if ENV['MESATESTHUB_API_KEY']
+        shell.say "Using the API key in MESATESTHUB_API_KEY (#{s.masked_api_key}). " \
+                  'Unset it to enter one here.', :yellow
+      else
+        response = shell.ask 'What is the MESATestHub API key for this ' \
+          "computer? (#{s.masked_api_key || 'none'}):", :blue
+        s.api_key = response.strip unless response.strip.empty?
+        next shell.say('An API key is required.', :red) unless s.api_key
+      end
+
+      check = s.confirm_computer(computer_name: nil)
+      if check['valid']
+        s.computer_name = check['computer'] || s.computer_name
+        shell.say "Key accepted: this computer is #{s.computer_name} on MESATestHub.", :green
+        return
+      elsif check['reachable'] == false
+        shell.say 'Saving the key unverified.', :yellow
+        response = shell.ask 'What is the name of this computer on ' \
+          "MESATestHub? (#{s.computer_name}):", :blue
+        s.computer_name = response unless response.empty?
+        return
+      end
+
+      shell.say check['message'], :red
+      if ENV['MESATESTHUB_API_KEY']
+        raise TestHubError, 'MESATESTHUB_API_KEY was refused; fix or unset it.'
+      end
+      s.api_key = nil
+    end
+  end
+
   def self.new_from_config(
     config_file: File.join(ENV['HOME'], '.mesa_test', 'config.yml'),
     force_setup: false,
@@ -107,6 +138,9 @@ e-mail and password will be stored in plain text.'
   )
     new_submitter = new(config_file: config_file, base_uri: base_uri)
     if force_setup
+      # Start the wizard from the current settings, including a 1.x
+      # config's (logs token, directories), so upgrading keeps them.
+      new_submitter.load_computer_data if File.exist? config_file
       new_submitter.setup
     elsif not File.exist? config_file
       puts "No such config file #{config_file}. Starting setup wizard."
@@ -116,45 +150,41 @@ e-mail and password will be stored in plain text.'
     return new_submitter
   end
 
-  attr_accessor :computer_name, :user_name, :email, :password, :platform,
-                :mesa_mirror, :mesa_work, :platform_version, :processor,
-                :config_file, :base_uri, :github_protocol, :logs_token
+  # Optional run modes a commit can request through its message
+  # (`[ci optional]`, `[ci fpe]`, `[ci converge]`), in MESATestHub's names.
+  CAPABILITIES = %i[full_inlists fpe converge].freeze
+
+  attr_accessor :computer_name, :user_name,
+                :mesa_mirror, :mesa_work, :processor,
+                :config_file, :base_uri, :github_protocol, :logs_token,
+                :capabilities
+  attr_writer :api_key
 
   attr_reader :shell
 
   # many defaults are set in body
   def initialize(
-      computer_name: nil, user_name: nil, email: nil, github_protocol: nil,
-      mesa_mirror: nil, platform: nil, platform_version: nil, processor: nil,
+      computer_name: nil, user_name: nil, github_protocol: nil,
+      mesa_mirror: nil, processor: nil,
       config_file: nil, base_uri: nil, logs_token: nil
   )
     @computer_name = computer_name || Socket.gethostname.scan(/^[^\.]+\.?/)[0]
     @computer_name.chomp!('.') if @computer_name
     @user_name = user_name || (ENV['USER'] || ENV['USERNAME'])
-    @email = email || ''
-    @password = password || ''
     @github_protocol = github_protocol || :ssh
     @mesa_mirror = mesa_mirror ||
       File.join(ENV['HOME'], '.mesa_test', 'mirror')
     @mesa_work = mesa_work ||
       File.join(ENV['HOME'], '.mesa_test', 'work')
-    @platform = platform
-    if @platform.nil?
-      @platform =
-        if OS.osx?
-          'macOS'
-        elsif OS.linux?
-          'Linux'
-        else
-          ''
-        end
-    end
-    @platform_version = platform_version || ''
     @processor = processor || ''
     @config_file = config_file || File.join(ENV['HOME'], '.mesa_test',
                                             'config.yml')
-    @base_uri = base_uri
+    # MESATESTHUB_URL points the client at another testhub (a local dev
+    # server, staging) without editing MODE in bin/mesa_test.
+    @base_uri = ENV['MESATESTHUB_URL'] || base_uri
     @logs_token = logs_token || ENV['MESA_LOGS_TOKEN']
+    @api_key = nil
+    @capabilities = CAPABILITIES.map { |c| [c, false] }.to_h
 
     # set up thor-proof way to get responses from user. Thor hijacks the
     # gets command, so we have to use its built-in "ask" method, which is
@@ -169,17 +199,52 @@ e-mail and password will be stored in plain text.'
     yield self if block_given?
   end
 
+  # The per-computer API key: MESATESTHUB_API_KEY from the environment wins
+  # over the config file, so a cluster job can supply it without the key
+  # ever being written to disk. nil means no key (see #require_api_key!).
+  def api_key
+    key = ENV['MESATESTHUB_API_KEY'] || @api_key
+    key.nil? || key.strip.empty? ? nil : key.strip
+  end
+
+  # Every MESATestHub request needs a key, so commands that talk to the hub
+  # call this first, before checking anything out or building.
+  def require_api_key!
+    return if api_key
+
+    raise TestHubError, <<~MSG
+      No MESATestHub API key is set. (This version of mesa_test no longer
+      signs in with your email and password.)
+
+      #{API_KEY_HELP}
+      Then run `mesa_test setup` and paste it when asked, or export
+      MESATESTHUB_API_KEY, which overrides the config file (e.g. to keep the
+      key off disk on a cluster).
+    MSG
+  end
+
+  def masked_api_key
+    key = api_key
+    key && "#{key[0, 10]}..."
+  end
+
+  # "fpe, full_inlists" => { full_inlists: true, fpe: true, converge: false }
+  def parse_capabilities(text)
+    wanted = text.downcase.split(/[\s,]+/).map(&:to_sym)
+    CAPABILITIES.map { |c| [c, wanted.include?(c)] }.to_h
+  end
+
   def confirm_computer_data
-    puts 'Ready to submit the following data:'
+    puts "This computer's mesa_test settings:"
     puts '-------------------------------------------------------'
     puts "Computer Name           #{computer_name}"
-    puts "User email              #{email}"
-    puts 'Password                ***********'
+    puts "API key                 #{masked_api_key || 'none'}"
+    puts "Optional test modes     #{capabilities.select { |_, on| on }.keys.join(', ').then { |c| c.empty? ? 'none' : c }}"
     puts "logs API token          #{logs_token}"
     puts "GitHub Protocol         #{github_protocol}"
     puts "MESA Mirror Location    #{mesa_mirror}"
     puts "MESA Work Location      #{mesa_work}"
-    puts "Platform                #{platform} #{platform_version}"
+    puts "OS (detected each run)  #{platform_version}"
     # puts "Config location         #{config_file}"
     puts '-------------------------------------------------------'
     puts ''
@@ -187,53 +252,92 @@ e-mail and password will be stored in plain text.'
     response.strip.casecmp('y').zero?
   end
 
-  # For one "computer" on the web server, and for [subjective] consistency
-  # reasons, the platform, processor, and RAM cannot be changed! If you
-  # change platforms (i.e. switch from mac to linux, or change between linux
-  # flavors), you should create a new computer account. Similarly, create new
-  # computer accounts if you change your RAM or processor. You do not need
-  # to change computers if you upgrade your platform (macOS 10.12 -> 10.13
   def save_computer_data
     data_hash = {
       'computer_name' => computer_name,
-      'email' => email,
-      'password' => password,
+      'api_key' => @api_key,
+      'capabilities' => capabilities.select { |_, on| on }.keys.map(&:to_s),
       'logs_token' => logs_token,
       'github_protocol' => github_protocol,
       'mesa_mirror' => mesa_mirror,
-      'mesa_work' => mesa_work,
-      'platform' => platform,
-      'platform_version' => platform_version,
+      'mesa_work' => mesa_work
     }
     # make sure there's a directory to write to
     unless dir_or_symlink_exists? File.dirname(config_file)
       FileUtils.mkdir_p File.dirname(config_file)
     end
-    File.open(config_file, 'w') { |f| f.write(YAML.dump(data_hash)) }
+    # Owner-only: the file holds the API key and logs token.
+    File.write(config_file, YAML.dump(data_hash), perm: 0o600)
+    File.chmod(0o600, config_file)
   end
 
+  # A setting missing from the file (e.g. capabilities in a 1.x config)
+  # keeps its default.
   def load_computer_data
-    data_hash = YAML.load(File.read(config_file))
-    @computer_name = data_hash['computer_name']
-    @email = data_hash['email']
-    @password = data_hash['password']
-    @logs_token = data_hash['logs_token']
-    @github_protocol = data_hash['github_protocol'].to_sym
-    @mesa_mirror = data_hash['mesa_mirror']
-    @mesa_work = data_hash['mesa_work']
-    @platform = data_hash['platform']
-    @platform_version = data_hash['platform_version']
+    data_hash = YAML.load(File.read(config_file)) || {}
+    @computer_name = data_hash.fetch('computer_name', @computer_name)
+    @api_key = data_hash.fetch('api_key', @api_key)
+    if data_hash.key?('capabilities')
+      @capabilities = parse_capabilities(Array(data_hash['capabilities']).join(','))
+    end
+    @logs_token = data_hash['logs_token'] || @logs_token
+    @github_protocol = (data_hash['github_protocol'] || @github_protocol).to_sym
+    @mesa_mirror = data_hash.fetch('mesa_mirror', @mesa_mirror)
+    @mesa_work = data_hash.fetch('mesa_work', @mesa_work)
+  end
+
+  # This computer's OS version, detected (once per run) rather than
+  # configured, since a typed-in value goes stale with the first OS update.
+  # MESATestHub shows it after the computer's platform (macOS / linux), so
+  # macOS sends just the number and Linux the distribution:
+  # "27.0.1", "Rocky Linux 8.10", "Ubuntu 24.04 (WSL)".
+  def platform_version
+    @platform_version ||= self.class.detect_platform_version
+  end
+
+  def self.detect_platform_version
+    version = OS.mac? ? `sw_vers -productVersion 2>/dev/null`.strip : linux_version
+    version = `uname -sr 2>/dev/null`.strip if version.to_s.empty?
+    version
+  rescue StandardError
+    `uname -sr 2>/dev/null`.strip rescue ''
+  end
+
+  # The freedesktop os-release standard (every current distribution, and
+  # containers; /usr/lib/os-release is its fallback location):
+  # NAME="Rocky Linux", VERSION_ID="8.10". Rolling releases (Arch) have no
+  # VERSION_ID. RHEL/CentOS 6 predate it but have /etc/redhat-release.
+  def self.linux_version
+    path = %w[/etc/os-release /usr/lib/os-release].find { |f| File.readable?(f) }
+    name =
+      if path
+        os = File.readlines(path, chomp: true)
+                 .map { |line| line.split('=', 2) }.select { |kv| kv.size == 2 }
+                 .to_h { |k, v| [k.strip, v.strip.delete('"\'')] }
+        [os['NAME'], os['VERSION_ID']].compact.join(' ')
+      elsif File.readable?('/etc/redhat-release')
+        # "CentOS release 6.10 (Final)"
+        File.read('/etc/redhat-release').strip.sub(/ \(.*\)\z/, '')
+      end
+    return name if name.to_s.empty?
+
+    wsl = File.readable?('/proc/version') && File.read('/proc/version').match?(/microsoft/i)
+    wsl ? "#{name} (WSL)" : name
   end
 
   # Parameters to be submitted in JSON format for reporting information about
-  # the submitting user and computer
+  # the submitting computer. The API key identifies it; the name is a
+  # cross-check (the hub refuses a key that belongs to another computer).
   def submitter_params
-    {
-      email: email,
-      password: password,
-      computer: computer_name,
-      platform_version: platform_version
-    }
+    { computer: computer_name, platform_version: platform_version }
+  end
+
+  # Headers for every MESATestHub request.
+  def testhub_headers(json_body: true)
+    headers = { 'Accept' => 'application/json' }
+    headers['Content-Type'] = 'application/json' if json_body
+    headers['Authorization'] = "Bearer #{api_key}" if api_key
+    headers
   end
 
   # Parameters to be submitted in JSON format for reporting information about
@@ -261,28 +365,55 @@ e-mail and password will be stored in plain text.'
   # test cases and the test results. These will be encoded as an array of
   # JSON objects.
   def instance_params(mesa)
-    has_errors = []
-    res = []
-    mesa.test_case_names.each do |mod, names|
-      names.each do |test_name|
-        begin
-          test_case = mesa.test_cases[mod][test_name]
-          res << test_case.results_hash
-        rescue TestCaseDirError
-          # shell.say "It appears that #{test_case.test_name} has not been "\
-          #           'run yet. Unable to submit data for this test.', :red
-          has_errors << test_case
+    submittable_tests(mesa).map(&:results_hash)
+  end
+
+  # Tests with results from *this* build: run, readable, and not older than
+  # the build itself. A reused work directory (`install --no-checkout`) can
+  # still hold testhub.yml files from an earlier build; those would be filed
+  # under the wrong commit, so they're skipped (and counted).
+  def submittable_tests(mesa)
+    built_at = mesa.built_at
+    unread = []
+    stale = 0
+    tests = mesa.test_case_names.flat_map do |mod, names|
+      names.filter_map do |test_name|
+        test_case = mesa.test_cases[mod][test_name]
+        next unless test_case&.ran?
+
+        if built_at && File.mtime(test_case.testhub_file) < built_at
+          stale += 1
+          next
         end
+        test_case.results_hash
+        test_case
+      rescue TestCaseDirError, Psych::Exception
+        unread << test_case
+        nil
       end
     end
-    unless has_errors.empty?
-      shell.say "The following test cases could NOT be read for submission:",
-                :red
-      has_errors.each do |test_case|
-        shell.say "- #{test_case.test_name}", :red
-      end
+    if stale.positive?
+      shell.say "Skipping #{stale} test result#{'s' unless stale == 1} older than this " \
+                'build (left over from an earlier one).', :yellow
     end
-    res
+    unless unread.empty?
+      shell.say 'The following test cases could NOT be read for submission:', :red
+      unread.each { |test_case| shell.say "- #{test_case.test_name}", :red }
+    end
+    tests
+  end
+
+  # The build's recorded run modes in the submissions API's `claim:` block
+  # (use_* flags; informational on the server, which reads the modes each
+  # test actually ran with from its testhub.yml).
+  def run_mode_params(mesa)
+    modes = mesa.recorded_run_modes
+    {
+      use_fpe: modes[:fpe] == true,
+      use_full_inlists: modes[:skip_optional] == false,
+      use_converge: modes[:resolution_factor].is_a?(String) ||
+                    modes[:resolution_factor].is_a?(Numeric)
+    }
   end
 
   # Parameters for reporting a failed compilation to the logs server
@@ -314,23 +445,39 @@ e-mail and password will be stored in plain text.'
     [test_case.results_hash]
   end
 
-  # Phone home to testhub and confirm that computer and user are valid. Useful
-  # for confirming that submissions will be accepted before wasting time on a
-  # test later.
-  def confirm_computer
+  # Phone home to testhub and confirm that the API key is valid (and, given a
+  # computer name, that it belongs to that computer). Useful for confirming
+  # that submissions will be accepted before wasting time on a test later.
+  # 'reachable' is false when the hub couldn't be asked at all.
+  def confirm_computer(computer_name: self.computer_name)
     uri = URI.parse(base_uri + '/check_computer.json')
     https = Net::HTTP.new(uri.hostname, uri.port)
     https.use_ssl = base_uri.include? 'https'
 
-    request = Net::HTTP::Post.new(
-      uri, initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' }
-    )
-    request.body = {
-      email: email,
-      password: password,
-      computer_name: computer_name
-    }.to_json
-    JSON.parse(https.request(request).body).to_hash
+    request = Net::HTTP::Post.new(uri, testhub_headers)
+    request.body = { computer_name: computer_name }.compact.to_json
+    response = testhub_request(https, request)
+    # if the hub was unreachable, behave as an unverified computer; the
+    # network error has already been reported by testhub_request
+    if response.nil?
+      return { 'valid' => false, 'reachable' => false,
+               'message' => 'Could not verify this computer with MESATestHub.' }
+    end
+
+    # Anything but a 200 (a 429 from the rate limiter, a 5xx, a proxy's HTML
+    # error page) carries no valid/message keys, so name the status and the
+    # server's error instead of failing with a blank line.
+    unless response.is_a?(Net::HTTPOK)
+      return { 'valid' => false,
+               'message' => 'Could not verify this computer with MESATestHub ' \
+                            "(HTTP #{response.code}: #{error_message(response)})." }
+    end
+
+    JSON.parse(response.body).to_hash
+  rescue JSON::ParserError
+    { 'valid' => false,
+      'message' => 'Could not verify this computer: MESATestHub sent an ' \
+                   'unreadable response.' }
   end
 
   # submit entire commit's worth of test cases, OR submit compilation status
@@ -344,10 +491,7 @@ e-mail and password will be stored in plain text.'
     https = Net::HTTP.new(uri.hostname, uri.port)
     https.use_ssl = true if base_uri.include? 'https'
 
-    request = Net::HTTP::Post.new(
-      uri,
-      initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' }
-    )
+    request = Net::HTTP::Post.new(uri, testhub_headers)
 
     # create the request body for submission to the submissions API
     # 
@@ -358,19 +502,32 @@ e-mail and password will be stored in plain text.'
     empty ||= !mesa.installed?
     request_data = {submitter: submitter_params,
                     commit: commit_params(mesa, empty: empty, entire: !empty)}
+    request_data[:claim] = run_mode_params(mesa) if mesa.recorded_run_modes.any?
+    # Fingerprints let MESATestHub ignore results it already has (a retried
+    # submit, a head node re-sending what compute nodes sent).
+    fingerprints = {}
+    fingerprints['build'] = mesa.build_fingerprint if mesa.build_fingerprint
     # don't need test instances if it's an empty submission or if compilation
     # failed
+    tests = []
     if !empty && request_data[:commit][:compiled]
-      request_data[:instances] = instance_params(mesa)
+      tests = submittable_tests(mesa)
+      request_data[:instances] = tests.map(&:results_hash)
+      tests.each { |t| fingerprints[t.fingerprint_key] = t.fingerprint }
     end
+    request_data[:fingerprints] = fingerprints unless fingerprints.empty?
     request.body = request_data.to_json
 
     # actually do the submission
-    response = https.request request
+    response = testhub_request(https, request)
 
-    if !response.is_a? Net::HTTPCreated
+    if response.nil?
+      # network failure; testhub_request already explained why
+      false
+    elsif !response.is_a? Net::HTTPCreated
       shell.say "\nFailed to submit some or all test case instances and/or "\
-                'commit data.', :red
+                "commit data (server responded #{response.code} "\
+                "#{response.message}).", :red
       false
     else
       shell.say "\nSuccessfully submitted commit #{mesa.sha}.", :green
@@ -382,15 +539,13 @@ e-mail and password will be stored in plain text.'
 
       # compilation succeded, so submit any logs for failing tests
       res = true
-      unless empty
-        mesa.test_cases.each do |mod, test_case_hash|
-          test_case_hash.each do |tc_name, test_case|
-            # get at each individual test case, see if it failed, and if it
-            # did, submit its log files
-            if !test_case.passed? || force_logs
-              res &&= submit_test_log(test_case, skip_passing: !force_logs)
-            end
-          end
+      # logs for the failing tests just submitted (or all, with force_logs)
+      tests.each do |test_case|
+        if !test_case.passed? || force_logs
+          # (not `res &&= submit_test_log(...)`: after one failed upload that
+          # would skip every remaining test's logs)
+          sent = submit_test_log(test_case, skip_passing: !force_logs)
+          res &&= sent
         end
       end
 
@@ -412,10 +567,7 @@ e-mail and password will be stored in plain text.'
     https = Net::HTTP.new(uri.hostname, uri.port)
     https.use_ssl = true if base_uri.include? 'https'
 
-    request = Net::HTTP::Post.new(
-      uri,
-      initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' }
-    )
+    request = Net::HTTP::Post.new(uri, testhub_headers)
 
     # create the request body for submission to the submissions API
     # 
@@ -423,16 +575,22 @@ e-mail and password will be stored in plain text.'
     # entire (... there is only test case)
     request_data = {submitter: submitter_params,
                     commit: commit_params(mesa, empty: false, entire: false),
-                    instances: single_instance_params(test_case)}
+                    instances: single_instance_params(test_case),
+                    fingerprints: { test_case.fingerprint_key => test_case.fingerprint }}
+    request_data[:claim] = run_mode_params(mesa) if mesa.recorded_run_modes.any?
     request.body = request_data.to_json
 
     # actually do the submission
-    response = https.request request
+    response = testhub_request(https, request)
 
-    if !response.is_a? Net::HTTPCreated
+    if response.nil?
+      # network failure; testhub_request already explained why
+      return false
+    elsif !response.is_a? Net::HTTPCreated
       shell.say "\nFailed to submit #{test_case.test_name} for commit "\
-                "#{mesa.sha}", :red
-      false
+                "#{mesa.sha} (server responded #{response.code} "\
+                "#{response.message}).", :red
+      return false
     else
       shell.say "\nSuccessfully submitted instance of #{test_case.test_name} "\
                 "for commit #{mesa.sha}.", :green
@@ -444,17 +602,50 @@ e-mail and password will be stored in plain text.'
     end
   end
 
+  # Perform an HTTP request against the test hub with bounded connect/read
+  # timeouts, so a slow or unreachable server (e.g. while it is under heavy
+  # load) fails fast with a clear message instead of hanging on the default
+  # 60-second connect timeout and then dumping a raw Ruby backtrace. Returns
+  # the Net::HTTPResponse, or +nil+ if the request could not be completed
+  # because of a network problem.
+  def testhub_request(https, request)
+    https.open_timeout = 10
+    https.read_timeout = 60
+    https.request(request)
+  rescue StandardError => e
+    shell.say "\nCould not reach the test hub at #{https.address} "\
+              "(#{e.class}: #{e.message}).", :red
+    nil
+  end
+
   # make generic request to LOGS server
   # +params+ is a hash of data to be encoded as JSON and sent off
+  #
+  # Returns the Net::HTTPResponse on success, or +nil+ if the request could
+  # not be completed because of a network problem (the LOGS server being
+  # unreachable, slow, or refusing connections). The LOGS server only receives
+  # diagnostic build/test output, so a failure here must never crash the run or
+  # fail a CI build whose actual test results already reached the test hub. We
+  # cap the connect/read time so we fail fast instead of hanging on the default
+  # 60-second open timeout for every test case.
   def submit_logs(params)
     #uri = URI('https://logs.mesastar.org/uploads')
     uri = URI('https://mesa-logs.flatironinstitute.org/uploads')
     https = Net::HTTP.new(uri.host, uri.port)
     https.use_ssl = true
+    https.open_timeout = 10
+    https.read_timeout = 30
     req = Net::HTTP::Post.new(uri.path, 'Content-Type' => 'application/json',
                               'X-Api-Key' => logs_token)
     req.body = params.to_json
-    https.request(req)
+    begin
+      https.request(req)
+    rescue StandardError => e
+      shell.say "\nCould not reach the LOGS server at #{uri.host} "\
+                "(#{e.class}: #{e.message}). Skipping log upload; this does "\
+                'not affect test results already sent to the test hub.', :yellow
+      nil
+    end
   end
 
   # send build log to the logs server
@@ -473,9 +664,13 @@ e-mail and password will be stored in plain text.'
     res = submit_logs(build_log_params(mesa))
 
     # report out results
-    if !res.is_a? Net::HTTPOK
+    if res.nil?
+      # network failure; submit_logs already explained why. Don't fail the run.
+      false
+    elsif !res.is_a? Net::HTTPOK
       shell.say "\nFailed to submit build.log to the LOGS server for commit "\
-                "#{mesa.sha}.", :red
+                "#{mesa.sha} (server responded #{res.code} #{res.message}).",
+                :red
       false
     else
       shell.say "\nSuccessfully submitted build.log to the LOGS server for "\
@@ -503,9 +698,13 @@ e-mail and password will be stored in plain text.'
     res = submit_logs(test_log_params(test_case))
 
     # report out results
-    if !res.is_a? Net::HTTPOK
+    if res.nil?
+      # network failure; submit_logs already explained why. Don't fail the run.
+      false
+    elsif !res.is_a? Net::HTTPOK
       shell.say "Failed to submit logs for test case #{test_case.test_name} "\
-                "in commit #{test_case.mesa.sha}.", :red
+                "in commit #{test_case.mesa.sha} (server responded "\
+                "#{res.code} #{res.message}).", :red
       false
     else
       shell.say "Successfully submitted logs for test case "\
@@ -515,9 +714,8 @@ e-mail and password will be stored in plain text.'
     end
   end
 
-  # GET a search query from the testhub. Credentials and the query string
-  # ride in the URL's query parameters (per the testhub search API contract;
-  # the controller also accepts session auth, but a CLI sends them in-band).
+  # GET a search query from the testhub. The query rides in the URL's
+  # query_text parameter; the API key in the Authorization header.
   # Returns the raw Net::HTTP response.
   def search(query_text)
     get_search('/test_instances/search.json', query_text)
@@ -529,23 +727,112 @@ e-mail and password will be stored in plain text.'
     get_search('/test_instances/search_count.json', query_text)
   end
 
+  # What a claimed run is running on, for MESATestHub to show on pending
+  # tests (so a long-running claim can be explained: an old machine on two
+  # threads, a different SDK). Only what's actually known is sent:
+  #   - with a built +mesa+, its testhub.yml compiler record (SDK, compiler,
+  #     math backend), which is accurate even if the shell's SDK changed
+  #   - otherwise (a build claim, made before compiling) just the SDK the
+  #     shell has active, from mesasdk_init.sh's MESASDK_VERSION
+  #   - OMP_NUM_THREADS and this computer's platform version
+  def claim_environment(mesa = nil)
+    env = {}
+    if mesa&.install_attempted?
+      env.merge!(mesa.compiler_hash.reject { |_k, v| v.to_s.strip.empty? || v == 'Unknown' })
+    end
+    sdk = ENV['MESASDK_VERSION'].to_s.strip
+    env[:sdk_version] ||= sdk unless sdk.empty?
+    threads = ENV['OMP_NUM_THREADS'].to_i
+    env[:omp_num_threads] = threads if threads.positive?
+    env[:platform_version] = platform_version unless platform_version.to_s.strip.empty?
+    env
+  end
+
+  # Ask MESATestHub what this computer should do next (POST /api/v1/dispatch).
+  # +scope+ is 'build' (which commit to build) or 'test' (which test to run;
+  # pass the +sha+ you built). +capabilities+ defaults to the configured
+  # optional modes; for a test, +fpe+ should say whether *this build* has
+  # FPE checks, since MESA_FPE_CHECKS_ON is a compile-time switch.
+  #
+  # Returns the recommendation as a Hash (commit_sha, branch, scope,
+  # test_case_module, test_case_name, flags, reasons, ...), or nil when
+  # there's nothing useful to do. Raises TestHubError on any other answer.
+  def dispatch(scope:, sha: nil, capabilities: self.capabilities)
+    body = { scope: scope }
+    body[:commit_sha] = sha if sha
+    CAPABILITIES.each { |c| body[:"can_#{c}"] = !!capabilities[c] }
+    response = post_testhub('/api/v1/dispatch', submitter: submitter_params, dispatch: body)
+    raise TestHubError, 'Dispatch failed: could not reach MESATestHub.' if response.nil?
+    return nil if response.is_a?(Net::HTTPNoContent)
+    return JSON.parse(response.body) if response.is_a?(Net::HTTPSuccess)
+
+    raise TestHubError, "Dispatch failed (HTTP #{response.code}): #{error_message(response)}"
+  end
+
+  # Tell MESATestHub this computer is about to build +sha+ (scope 'build') or
+  # run a test on it (scope 'test' with +mod+ and +test_case+, or
+  # +all_test_cases: true+ for the whole suite). The commit then shows as
+  # pending instead of untested. The later submission fulfills the claim by
+  # matching computer, commit, and test, so nothing needs remembering here.
+  #
+  # Claims are advisory: a failure is reported and swallowed (returns nil) so
+  # it never stops a test run.
+  #
+  # The claim also says what the run is running on (see #claim_environment),
+  # which MESATestHub shows on the pending test. Pass +mesa+ for test claims
+  # so the build's own compiler record is used.
+  def claim(sha:, scope:, mod: nil, test_case: nil, all_test_cases: false,
+            flags: {}, dispatched_at: nil, mesa: nil)
+    body = { commit_sha: sha, scope: scope }
+    body.merge!(claim_environment(scope.to_s == 'test' ? mesa : nil))
+    body[:test_case_module] = mod.to_s if mod
+    body[:test_case_name] = test_case if test_case
+    body[:all_test_cases] = true if all_test_cases
+    body[:dispatched_at] = dispatched_at if dispatched_at
+    CAPABILITIES.each { |c| body[:"use_#{c}"] = !!flags[c] }
+    response = post_testhub('/api/v1/claims', submitter: submitter_params, claim: body)
+    return nil if response.nil? # unreachable; testhub_request said so
+    return JSON.parse(response.body) if response.is_a?(Net::HTTPCreated)
+
+    shell.say "Could not claim #{claim_label(scope, mod, test_case, all_test_cases)} " \
+              "(HTTP #{response.code}: #{error_message(response)}); continuing anyway.",
+              :yellow
+    nil
+  end
+
   private
 
+  # POST JSON to the hub with testhub_request's bounded timeouts. Returns
+  # the response, or nil after a network failure (already reported).
+  def post_testhub(path, payload)
+    uri = URI.parse(base_uri + path)
+    https = Net::HTTP.new(uri.hostname, uri.port)
+    https.use_ssl = base_uri.include? 'https'
+    request = Net::HTTP::Post.new(uri, testhub_headers)
+    request.body = payload.to_json
+    testhub_request(https, request)
+  end
+
+  def error_message(response)
+    JSON.parse(response.body.to_s)['error'] || response.message
+  rescue JSON::ParserError
+    response.message
+  end
+
+  def claim_label(scope, mod, test_case, all_test_cases)
+    return 'the build' if scope.to_s == 'build'
+    return 'every test' if all_test_cases
+    "#{mod}/#{test_case}"
+  end
+
   def get_search(path, query_text)
     uri = URI.parse(base_uri + path)
-    uri.query = URI.encode_www_form(
-      email: email,
-      password: password,
-      query_text: query_text
-    )
+    uri.query = URI.encode_www_form(query_text: query_text)
 
     https = Net::HTTP.new(uri.hostname, uri.port)
     https.use_ssl = base_uri.include? 'https'
 
-    request = Net::HTTP::Get.new(
-      uri,
-      initheader = { 'Accept' => 'application/json' }
-    )
+    request = Net::HTTP::Get.new(uri, testhub_headers(json_body: false))
 
     https.request(request)
   end
@@ -738,23 +1025,65 @@ class Mesa
     b64_file(build_log)
   end
 
+  COMPILER_KEYS = %i[compiler compiler_version sdk_version math_backend].freeze
+
   # sourced from $MESA_DIR/testhub.yml, which should be created after
-  # installation
+  # installation. Only the compiler keys are taken: the file also carries
+  # mesa_test's own run-mode block (see #record_run_modes), which doesn't
+  # belong in the commit payload.
   def compiler_hash
-    data_file = File.join(mesa_dir, 'testhub.yml')
     res = {
             compiler: 'Unknown',
             sdk_version: 'Unknown',
             math_backend: 'Unknown'
           }
-    if File.exist? data_file
-      res = res.merge(YAML.safe_load(File.read(data_file)) || {})
-      # currently version_number is reported, but we don't need that in Git land
-      res.delete('version_number') # returns the value, not the updated hash
-      res
+    # testhub.yml has string keys; symbolize them so they replace the
+    # defaults above rather than sitting beside them as duplicate keys
+    # (which json >= 3 refuses to serialize).
+    found = install_data.transform_keys(&:to_sym).slice(*COMPILER_KEYS)
+    res.merge(found)
+  end
+
+  # The key under which `mesa_test install` records, in $MESA_DIR/testhub.yml,
+  # the run modes this build was made for. The file is written by MESA's own
+  # install (math/test), so it is recreated on every build and can't go stale.
+  RUN_MODES_KEY = 'mesa_test_run_modes'.freeze
+
+  # Run modes recorded at install, as a hash with any of
+  #   fpe:               true/false (compiled with MESA_FPE_CHECKS_ON?)
+  #   skip_optional:     true/false (MESA_SKIP_OPTIONAL set or unset)
+  #   resolution_factor: a factor string, or false for "unset"
+  # Only modes that were explicitly chosen at install are present; anything
+  # absent falls back to the caller's environment.
+  def recorded_run_modes
+    (install_data[RUN_MODES_KEY] || {}).transform_keys(&:to_sym)
+                                       .slice(*RUN_MODE_KEYS)
+  end
+
+  # Append the modes chosen for this build to testhub.yml. Call after the
+  # install, which rewrites the file.
+  def record_run_modes(modes)
+    data_file = File.join(mesa_dir, 'testhub.yml')
+    return unless File.exist?(data_file)
+
+    modes = modes.slice(*RUN_MODE_KEYS)
+    return if modes.empty?
+
+    block = { RUN_MODES_KEY => modes.transform_keys(&:to_s) }.to_yaml.sub(/\A---\n/, '')
+    File.open(data_file, 'a') do |f|
+      f.puts '# Added by mesa_test install: the run modes this build was made for.'
+      f.puts '# `mesa_test test` applies them unless told otherwise.'
+      f.write block
     end
   end
 
+  def install_data
+    data_file = File.join(mesa_dir, 'testhub.yml')
+    return {} unless File.exist?(data_file)
+
+    YAML.safe_load(File.read(data_file)) || {}
+  end
+
   ## TEST SUITE METHODS
 
   def check_mod(mod)
@@ -845,6 +1174,18 @@ class Mesa
     )
   end
 
+  # Fingerprint of this build (a hash of build.log), so MESATestHub can
+  # recognize a build report it already has. nil before any build.
+  def build_fingerprint
+    file_fingerprint(File.join(mesa_dir, 'build.log'))
+  end
+
+  # When this build finished (build.log's mtime), or nil.
+  def built_at
+    build_log = File.join(mesa_dir, 'build.log')
+    File.exist?(build_log) ? File.mtime(build_log) : nil
+  end
+
   def install_attempted?
     File.exist? File.join(mesa_dir, 'testhub.yml')
   end
@@ -1019,6 +1360,23 @@ class MesaTestCase
     File.join(test_case_dir, 'testhub.yml')
   end
 
+  # Run this test like #do_one, but as its own subprocess with its own
+  # working directory and environment (+env+ overrides; nil unsets), so
+  # several can run at once -- #do_one changes the process-wide directory
+  # and expects the environment to be set globally. Output goes to
+  # each_test_run.log in the test's directory. Returns true if
+  # each_test_run exited cleanly (the test's pass/fail is in testhub.yml).
+  def run_isolated(env: {})
+    log = File.join(test_case_dir, 'each_test_run.log')
+    pid = Process.spawn(env, './each_test_run', position.to_s,
+                        chdir: test_suite_dir, out: log, err: %i[child out], pgroup: true)
+    RUNNING_TESTS_LOCK.synchronize { RUNNING_TESTS[pid] = test_name }
+    _, status = Process.wait2(pid)
+    status.success?
+  ensure
+    RUNNING_TESTS_LOCK.synchronize { RUNNING_TESTS.delete(pid) } if pid
+  end
+
   # just punt to +each_test_run+ in the test_suite directory. It's your problem
   # now, sucker!
   def do_one
@@ -1036,6 +1394,23 @@ class MesaTestCase
     YAML.load(File.read(testhub_file))
   end
 
+  # How MESATestHub keys this test's fingerprint: "<module>/<test>".
+  def fingerprint_key
+    "#{mod}/#{test_name}"
+  end
+
+  # Fingerprint of this run's results (a hash of testhub.yml).
+  def fingerprint
+    file_fingerprint(testhub_file)
+  end
+
+  # Whether this test wrote results at or after +time+, i.e. during the
+  # run that started then (not a testhub.yml left from an earlier run).
+  # mtimes can be whole seconds, so compare at that resolution.
+  def ran_since?(time)
+    ran? && File.mtime(testhub_file) >= time.floor
+  end
+
   # rough proxy for whether or not the test has even been run
   def ran?
     File.exist?(testhub_file)
@@ -1170,6 +1545,137 @@ def bashticks(command)
   `bash -c "#{command}"`.chomp
 end
 
+# Run modes: how MESA is told to run a test, and the environment variables
+# behind them. MESA's each_test_run records each one in the test's
+# testhub.yml (fpe_checks / run_optional / resolution_factor), which is what
+# MESATestHub reads.
+#
+#   fpe:               MESA_FPE_CHECKS_ON=1. Compile-time as well as run-time
+#                      (MESA's make reads it), so it belongs to the build.
+#   skip_optional:     MESA_SKIP_OPTIONAL set. MESA runs every inlist unless
+#                      this is set.
+#   resolution_factor: MESA_TEST_SUITE_RESOLUTION_FACTOR, which is what
+#                      `[ci converge]` sets.
+#
+# A mode that is nil/absent leaves the variable as the user's shell has it.
+RUN_MODE_KEYS = %i[fpe skip_optional resolution_factor].freeze
+DEFAULT_CONVERGE_FACTOR = '0.8'.freeze
+
+# ENV overrides for +modes+ (nil value = unset the variable).
+def run_mode_env(modes)
+  env = {}
+  env['MESA_FPE_CHECKS_ON'] = modes[:fpe] ? '1' : nil unless modes[:fpe].nil?
+  unless modes[:skip_optional].nil?
+    env['MESA_SKIP_OPTIONAL'] = modes[:skip_optional] ? 't' : nil
+  end
+  unless modes[:resolution_factor].nil?
+    env['MESA_TEST_SUITE_RESOLUTION_FACTOR'] =
+      modes[:resolution_factor] ? modes[:resolution_factor].to_s : nil
+  end
+  env
+end
+
+# The resolution factor a converge run should use: the user's own, if their
+# shell sets one, else DEFAULT_CONVERGE_FACTOR.
+def converge_factor
+  ENV['MESA_TEST_SUITE_RESOLUTION_FACTOR'] || DEFAULT_CONVERGE_FACTOR
+end
+
+# Run modes a MESATestHub dispatch asked for. Requests only ever *add* to the
+# user's setup: a mode the hub didn't ask for is left out (so the shell
+# decides), never forced off.
+#
+# A converge request may name its factor (`[ci converge 0.5]` arrives as
+# rec['converge_factor']). That's the commit author's request, so it wins
+# over +preferred+ -- a factor from --resolution-factor or the shell's
+# MESA_TEST_SUITE_RESOLUTION_FACTOR -- with a warning when they differ.
+# Without a named factor, +preferred+ (else the default) is used.
+def dispatch_run_modes(rec, preferred: nil, preferred_source: nil)
+  flags = rec['flags'] || {}
+  modes = {}
+  modes[:fpe] = true if flags['use_fpe']
+  modes[:skip_optional] = false if flags['use_full_inlists']
+  if flags['use_converge']
+    requested = rec['converge_factor']
+    preferred ||= ENV['MESA_TEST_SUITE_RESOLUTION_FACTOR']
+    preferred_source ||= 'MESA_TEST_SUITE_RESOLUTION_FACTOR'
+    if requested
+      factor = requested.to_s
+      if preferred && preferred.to_f != requested.to_f
+        warn "MESATestHub asked for resolution factor #{factor} ([ci converge #{factor}]); " \
+             "using it instead of your #{preferred_source}=#{preferred}."
+      end
+      modes[:resolution_factor] = factor
+    else
+      modes[:resolution_factor] = preferred || DEFAULT_CONVERGE_FACTOR
+    end
+  end
+  modes
+end
+
+# Process groups of tests started by MesaTestCase#run_isolated, so an
+# interrupted parallel run can stop them all.
+RUNNING_TESTS = {}
+RUNNING_TESTS_LOCK = Mutex.new
+
+def stop_running_tests
+  pids = RUNNING_TESTS_LOCK.synchronize { RUNNING_TESTS.keys }
+  pids.each do |pid|
+    Process.kill('TERM', -pid)
+  rescue Errno::ESRCH, Errno::EPERM
+    nil
+  end
+end
+
+# Run +jobs+ worker threads, each calling the block with its index until
+# the block returns :done. On Ctrl-C, kills the workers first (so none
+# goes on to submit a half-run test), then every running test, then
+# re-raises. Exceptions in a worker propagate when it's joined.
+def run_workers(jobs)
+  workers = []
+  jobs.times do |i|
+    workers << Thread.new do
+      Thread.current.report_on_exception = false
+      loop { break if yield(i) == :done }
+    end
+  end
+  workers.each(&:join)
+rescue Interrupt
+  workers.each(&:kill)
+  stop_running_tests
+  raise
+end
+
+# Hand each of +items+ to the block, +jobs+ at a time.
+def parallel_each(items, jobs)
+  queue = Queue.new
+  items.each { |item| queue << item }
+  run_workers([jobs, items.size].min) do
+    item = begin
+      queue.pop(true)
+    rescue ThreadError
+      next :done
+    end
+    yield item
+    nil
+  end
+end
+
+# Run the block with +overrides+ applied to ENV (nil deletes a variable),
+# restoring the previous values afterwards.
+def with_env(overrides)
+  saved = overrides.keys.map { |k| [k, ENV[k]] }.to_h
+  overrides.each { |k, v| v.nil? ? ENV.delete(k) : ENV[k] = v }
+  yield
+ensure
+  saved&.each { |k, v| v.nil? ? ENV.delete(k) : ENV[k] = v }
+end
+
+# SHA-256 of a file's contents, or nil if it doesn't exist.
+def file_fingerprint(path)
+  File.exist?(path) ? Digest::SHA256.file(path).hexdigest : nil
+end
+
 # encode the contents of a file as base-64
 def b64_file(filename)
   Base64.encode64(File.open(filename).read)