mesa_test 1.2.1 → 2.0.0

data/bin/mesa_test

@@ -15,37 +15,112 @@ end
 
 require 'thor'
 require 'yaml'
+require 'etc'
 
 class MesaTest < Thor
   DEVELOPMENT_URI = 'http://localhost:3000'.freeze
   STAGING_URI = 'https://beta-testhub.herokuapp.com'.freeze
 
+  # Exit status when MESATestHub has nothing for this computer to do, so a
+  # script can stop instead of re-testing whatever is already in the work
+  # directory:  mesa_test install best || exit
+  NOTHING_TO_DO_EXIT = 3
+
+  def self.exit_on_failure?
+    true
+  end
+
+  # --claim / --no-claim, shared by every command that starts work.
+  def self.claim_option
+    option :claim, desc: 'Tell MESATestHub this computer is working on the ' \
+                         'commit (and its tests) so it shows as pending, not ' \
+                         'untested. Use --no-claim if something else already ' \
+                         'claimed it.',
+                   type: :boolean, default: true
+  end
+
+  # Parallel testing (opt-in; with neither set, tests run serially in your
+  # shell's environment exactly as before).
+  def self.parallel_options
+    option :jobs, desc: 'Run this many tests at once (each in its own ' \
+                        'process; output in each_test_run.log in the test ' \
+                        'directory). Default 1: one after another.',
+                  type: :numeric, default: 1, aliases: '-j'
+    option :test_threads, desc: 'OMP_NUM_THREADS for each test. Default: ' \
+                                "your shell's OMP_NUM_THREADS (else all cores) " \
+                                'split evenly across --jobs.',
+                          type: :numeric
+  end
+
+  # NPROCS (MESA's install runs make -j$NPROCS; all cores if unset) and
+  # OMP_NUM_THREADS for the build.
+  def self.build_threads_option
+    option :build_threads, desc: 'Cores for compiling MESA (sets NPROCS and ' \
+                                 'OMP_NUM_THREADS for the install; default: ' \
+                                 'all cores).',
+                           type: :numeric
+  end
+
+  # The run-mode switches. Each is three-way: --x, --no-x, or unset. What
+  # unset means depends on the command (see each command's help).
+  def self.run_mode_options
+    option :skip_optional, desc: 'Skip optional inlists (sets MESA_SKIP_OPTIONAL); ' \
+                                 '--no-skip-optional runs them all, as MESA does ' \
+                                 'by default.',
+                           type: :boolean
+    option :fpe, desc: 'Floating-point exception checks (MESA_FPE_CHECKS_ON=1). ' \
+                       'Compiled in, so it is chosen when MESA is installed.',
+                 type: :boolean
+    option :converge, desc: 'Convergence run (sets MESA_TEST_SUITE_RESOLUTION_FACTOR, ' \
+                            "keeping your value if set, else #{DEFAULT_CONVERGE_FACTOR}).",
+                      type: :boolean
+    option :resolution_factor, desc: 'Convergence run at this factor (implies --converge). ' \
+                                     'With `best`, a factor the commit names ' \
+                                     '([ci converge 0.5]) wins.',
+                               type: :numeric
+  end
+
   desc 'test [TEST CASE]', 'run and submit a test case or commit'
   long_desc <<-LONGDESC
-    If you specify an optional test case, only that test case
-    will be tested. Otherwise all test cases in your work directory (set up in
-    your .mesa_test.yml file). TEST_CASE can also be an integer corresponding
-    to the position it has in do1_test_source (or a concatenated version of
-    that file if no module is specified). Modules are searched/concatenated in
-    this order:
+    Runs TEST_CASE, or with no TEST_CASE every test case, in the MESA
+    installation in your work directory (mesa_work in ~/.mesa_test/config.yml,
+    or --mesadir). TEST_CASE can be a name or an integer: its position in
+    do1_test_source (or in the modules' lists concatenated, if no --module is
+    given). Modules are searched/concatenated in this order:
 
     #{MesaTestCase.modules.map { |mod| '    ' + mod.to_s }.join(', ')}
+
+    Each test is claimed on MESATestHub as it starts, so it shows as pending,
+    and its result is submitted as soon as it finishes. This needs an API key
+    (see `mesa_test help setup`). --no-submit runs tests locally only:
+    nothing is claimed or sent, and no key is needed.
+
+    With no TEST_CASE, --jobs N runs N tests at once.
+
+    Run modes (--skip-optional, --fpe, --converge): a flag given here wins;
+    otherwise the mode recorded when this build was installed (in
+    $MESA_DIR/testhub.yml) applies; otherwise your shell's environment
+    does. FPE checks are compiled in, so a build installed with or without
+    them keeps that setting whatever is passed here.
   LONGDESC
 
-  option :force, desc: 'When submitting, assume submission data in ' \
-                       '~/.mesa_test.yml are correct rather than checking ' \
-                       'with user.',
+  option :force, desc: 'Use ~/.mesa_test/config.yml as it is. --no-force ' \
+                       'shows it and asks you to confirm (or fix) it first.',
                  type: :boolean, default: true, aliases: '-f'
   option 'force-logs', desc: 'Submit logs for test case(s), even if they pass',
                        type: :boolean, default: false
-  option :submit, desc: 'Submit results (if any) to MesaTestHub.',
+  option :submit, desc: 'Submit results (if any) to MESATestHub.',
                   type: :boolean, default: true
   option :module, desc: 'Which module to test and/or submit, if applicable.',
                   type: :string, default: 'all', aliases: '-m'
   option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
                    type: :string
+  claim_option
+  run_mode_options
+  parallel_options
 
   def test(test_case_name = nil)
+    validate_options!
     s = if options[:submit]
           create_and_check_submitter(force: options[:force])
         else
@@ -57,58 +132,57 @@ class MesaTest < Thor
       mod: options[:module].downcase.to_sym,
       github_protocol: s.github_protocol
     )
-    m.with_mesa_dir do
-      if test_case_name
-        # only testing one test case
-        t = create_and_check_test_case(mesa: m, test_case_name: test_case_name,
-                                       mod: options[:module].downcase.to_sym)
-  
-        # run test
-        t.do_one
-  
-        # bail out if not submitting
-        return unless options[:submit]
-  
-        # submit results
-        shell.say 'Submitting results to ' + s.base_uri + '... ', :blue
-        s.submit_instance(m, t, force_logs: options['force-logs'])
-      else
-        # run all tests
-        m.each_test_run(mod: options[:module].downcase.to_sym)
-  
-        # bail out if not submitting
-        return unless options[:submit]
-  
-        shell.say 'Submitting results to ' + s.base_uri + '... ', :blue
-        s.submit_commit(m, force_logs: options['force-logs'])
+    modes = test_run_modes(m)
+    with_env(run_mode_env(modes).merge(test_thread_env)) do
+      m.with_mesa_dir do
+        if test_case_name
+          # only testing one test case
+          t = create_and_check_test_case(mesa: m, test_case_name: test_case_name,
+                                         mod: options[:module].downcase.to_sym)
+
+          if options[:submit] && options[:claim]
+            s.claim(sha: m.sha, scope: 'test', mod: t.mod, test_case: t.test_name,
+                    flags: claim_flags(modes), mesa: m)
+          end
+
+          say_step(shell, "Testing #{t.mod}/#{t.test_name}")
+          started = Time.now
+          t.do_one
+          finish_test(s, m, t, started, submit: options[:submit])
+        else
+          # Each test is claimed as it starts and submitted as it finishes.
+          # (`install` reports the build.)
+          run_tests(s, m, mod: options[:module].downcase.to_sym,
+                          submit: options[:submit], flags: claim_flags(modes))
+        end
       end
     end
-    shell.say "Done.\n", :green
   end
 
   desc 'submit [TEST CASE]', 'submit a test case or commit'
   long_desc <<-LONGDESC
-    All tests are assumed to be in the default mesa directory (set as mesa_work
-    in .mesa_test.yml. If you specify an optional test case, only that test case
-    will be submitted. TEST_CASE can also be an integer corresponding to the
-    position it has in do1_test_source (or a concatenated version of that file
-    if no module is specified). Modules are searched/concatenated in this
-    order:
+    Submits results already in your work directory (mesa_work in
+    ~/.mesa_test/config.yml, or --mesadir) without running anything: say, from
+    a cluster's head node after jobs ran with --no-submit, or to retry after a
+    network failure. With TEST_CASE (a name or number, as for `test`), just
+    that test's result. Without it, the build status plus every test with
+    results from this build (or with --empty, just the build status), in one
+    request. Results left over from an earlier build are skipped, and
+    MESATestHub ignores results it already has, so submitting again is safe.
+    Needs an API key (see `mesa_test help setup`). Modules are
+    searched/concatenated in this order:
 
     #{MesaTestCase.modules.map { |mod| '    ' + mod.to_s }.join(', ')}
   LONGDESC
 
-  option :force, desc: 'When submitting, assume submission data in ' \
-                       '~/.mesa_test.yml are correct rather than checking ' \
-                       'with user.',
+  option :force, desc: 'Use ~/.mesa_test/config.yml as it is. --no-force ' \
+                       'shows it and asks you to confirm (or fix) it first.',
                  type: :boolean, default: true, aliases: '-f'
   option 'force-logs', desc: 'Submit logs for test case(s), even if they pass',
                        type: :boolean, default: false
   option :module, desc: 'Which module to test and/or submit, if applicable.',
                   type: :string, default: 'all', aliases: '-m'
-  option :empty, desc: 'Whether any test cases will be submitted. Useful for '\
-                       'indicating compilation status independently of ' \
-                       'single test submissions.',
+  option :empty, desc: 'Submit only the build status, no test results.',
                  type: :boolean, default: false, aliases: '-e'
   option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
                    type: :string
@@ -133,36 +207,32 @@ class MesaTest < Thor
   
         t = create_and_check_test_case(mesa: m, test_case_name: test_case_name,
                                        mod: options[:module].downcase.to_sym)
-        # submit results
-        shell.say 'Submitting results to ' + s.base_uri + '... ', :blue
+        raise Thor::Error, "#{t.mod}/#{t.test_name} has no results to submit." unless t.ran?
+
         s.submit_instance(m, t, force_logs: options['force-logs'])
-        shell.say "Done.\n", :green
       else
-        # submitting compilation information and either all tests (entire) or
-        # no tests (empty)
+        # The build status plus (unless --empty) every test with results from
+        # this build, in one request. MESATestHub stores it as a build row
+        # plus one row per test and ignores anything it already has (by
+        # fingerprint), so re-running `submit` is safe.
         begin
           m.check_installation
         rescue MesaDirError
-          shell.say %q{This MESA installation doesn't seem to be compiled } \
-            'properly. Attempting to submit a compilation failure to '\
-            'MESATestHub.', :red
+          shell.say "This build didn't finish compiling, so only the failed build " \
+                    'will be reported.', :yellow
           empty = true
-        ensure
-          # submit all tests
-          s.submit_commit(m, empty: empty, force_logs: options['force-logs'])
         end
+        s.submit_commit(m, empty: empty, force_logs: options['force-logs'])
       end
     end
   end
 
   desc 'checkout [SHA]', 'checkout a commit'
   long_desc <<-LONGDESC
-  Checks out a particular commit to default mesa directory (mesa_work in 
-  .mesa_test.yml). Uses git worktree magic to avoid re-downloading all of
-  mesa history if a mirror repo is already set up. Does not affect the
-  user's MESA_DIR or other environment variables.
-
-  If no sha is provided, defaults to main (if checkout is occuring).
+  Checks out SHA (default: main) into your work directory (mesa_work in
+  ~/.mesa_test/config.yml, or --mesadir), using git worktrees off the mirror
+  so MESA's history isn't downloaded again. Doesn't build, doesn't contact
+  MESATestHub, and doesn't touch your own MESA_DIR or environment.
   LONGDESC
 
   option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
@@ -180,16 +250,30 @@ class MesaTest < Thor
 
   desc 'install [SHA]', 'checkout and install a commit'
   long_desc <<-LONGDESC
-  Optionally checks out a particular commit to default mesa directory 
-  (mesa_work in .mesa_test.yml). Then runs installation script in the work
-  directory. Equivalent to an [optional] git checkout followed by going
-  into the directory and running ./clean and ./install, but there is some git
-  worktree magic on the backend to improve efficiency. SDK or compilers must be
-  set up prior. Does not affect the user's MESA_DIR or other environment
-  variables outside of execution.
-
-  If no sha is provided, defaults to main (if checkout is occuring). SHA is
-  ignored if using option --no-checkout
+  Checks out SHA (default: main) into your work directory (mesa_work in
+  ~/.mesa_test/config.yml, or --mesadir) and builds it: ./clean and
+  ./install, with git worktrees off the mirror so MESA's history isn't
+  downloaded again. The SDK or compilers must already be set up. Doesn't
+  touch your own MESA_DIR or environment. With --no-checkout, builds what's
+  already there and ignores SHA.
+
+  The build is claimed and its result (pass or fail, with the build log)
+  reported to MESATestHub, which needs an API key (see `mesa_test help
+  setup`). A failed build exits with status 1. --no-submit just builds, for
+  local work: nothing is claimed or sent, and no key is needed.
+
+  Run modes given here (--skip-optional, --fpe, --converge) are used for
+  the build and recorded in $MESA_DIR/testhub.yml, so later `mesa_test test`
+  runs on this build (e.g. cluster array jobs) apply them automatically.
+
+  If SHA is `best`, MESATestHub picks the commit this computer should build
+  next, and the run modes come from its request instead: the mode flags then
+  say what this computer is *willing* to do (overriding `capabilities` in the
+  config file). If nothing needs building, mesa_test exits with status
+  #{NOTHING_TO_DO_EXIT} without touching the work directory, so a script can
+  stop:
+
+      mesa_test install best || exit
   LONGDESC
 
   option :checkout, desc: 'Whether a checkout needs to happen. Use '\
@@ -197,12 +281,30 @@ class MesaTest < Thor
                     type: :boolean, default: true, aliases: '-c'
   option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
                    type: :string
+  option :submit, desc: 'Report the build (claim it first, then submit its ' \
+                        'status) to MESATestHub. --no-submit just builds, ' \
+                        'e.g. for local testing.',
+                  type: :boolean, default: true
+  claim_option
+  run_mode_options
+  build_threads_option
 
   def install(sha = 'main')
+    validate_options!
     # assuming config wizard has already been set (perhaps a little reckless)
     # so that we can get at the work directory and mirror directory
-    s = create_submitter(force: true)
-    m = if options[:checkout]
+    s = if options[:submit] || sha == 'best'
+          create_and_check_submitter(force: true)
+        else
+          create_submitter(force: true)
+        end
+    rec = nil
+    if sha == 'best'
+      rec = dispatch_build_or_exit(s)
+      sha = rec['commit_sha']
+    end
+    modes = rec ? modes_for(rec) : explicit_run_modes
+    m = if options[:checkout] || rec
           Mesa.checkout(
             sha: sha,
             work_dir: work_dir(options[:mesadir], s.mesa_work),
@@ -217,36 +319,51 @@ class MesaTest < Thor
             github_protocol: s.github_protocol
           )
         end
-    begin
-      m.clean
-      m.install
+    if options[:submit] && options[:claim]
+      s.claim(sha: m.sha, scope: 'build', flags: claim_flags(modes),
+              dispatched_at: rec && rec['dispatched_at'])
+    end
+    built = begin
+      build_with_modes(m, modes, raise_on_failure: true)
+      true
     rescue MesaDirError
-      shell.say "\nFailed in compiling MESA.", :red
-    else
-      shell.say "\nSuccessfully compiled MESA commit #{m.sha}.", :green
+      false
     end
+    # Report the build either way (a failure carries its build log), like
+    # `test` reports each test.
+    submit_build(s, m) if options[:submit]
+    exit 1 unless built
   end
 
   desc 'install_and_test [SHA]', 'Install, test, and submit an entire commit.'
   long_desc <<-LONGDESC
-  Optionally checks out a particular commit (specified by SHA, which default to
-  HEAD if not provided) to a mesa directory (mesa_work in .mesa_test.yml, 
-  unless a custom location is provided by --workdir=MYDIR). Then go into the
-  directory and run ./clean and ./install. SDK or compilers must be set up
-  prior. Once installation is complete, run the test suite, and report results
-  (commit and all test data) to MesaTestHub in a single request.
-  Does not affect the user's MESA_DIR or other environment variables. This is
-  basically a shortcut for running
-
-    `mesa_test install SOME_SHA`
-
-  followed by
-
-    `mesa_test test`
-
-  If no sha is provided, defaults to main (if checkout is occuring). SHA is
-  ignored if using option --no-checkout
-
+  Checks out SHA (default: main) into your work directory (mesa_work in
+  ~/.mesa_test/config.yml, or --mesadir), builds it (./clean and ./install;
+  the SDK or compilers must already be set up), and runs the test suite.
+  Doesn't touch your own MESA_DIR or environment. Roughly a shortcut for
+
+    mesa_test install SHA && mesa_test test
+
+  The build status is submitted as soon as the build finishes (a failed
+  build is reported and exits 1), then each test is claimed as it starts
+  and submitted as it finishes, so results roll in during the run. This
+  needs an API key (see `mesa_test help setup`). --no-submit builds and
+  tests locally without claiming or sending anything. With --no-checkout,
+  the build in the work directory is used and SHA is ignored.
+
+  --jobs N runs N tests at once; see also --test-threads and --build-threads.
+
+  Run modes (--skip-optional, --fpe, --converge) apply to the build and
+  every test; unset ones are left to your shell's environment.
+
+  If SHA is `best`, MESATestHub picks the commit (one this computer hasn't
+  tested yet) and the run modes (FPE checks, plus full inlists or a
+  convergence run if the commit asks for one); everything else is the
+  ordinary run above. The mode flags then say what this computer is
+  willing to do (overriding `capabilities` in the config file): commits
+  asking for [ci optional] or [ci converge] get those runs only if it's
+  willing. --no-submit doesn't apply in this mode. If nothing needs
+  testing, mesa_test exits with status #{NOTHING_TO_DO_EXIT}.
   LONGDESC
 
   option :checkout, desc: 'Whether a checkout needs to happen. Use '\
@@ -254,18 +371,24 @@ class MesaTest < Thor
                     type: :boolean, default: true, aliases: '-c'
   option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
                    type: :string
-  option :force, desc: 'When submitting, assume submission data in ' \
-                       '~/.mesa_test.yml are correct rather than checking ' \
-                       'with user.',
+  option :force, desc: 'Use ~/.mesa_test/config.yml as it is. --no-force ' \
+                       'shows it and asks you to confirm (or fix) it first.',
                  type: :boolean, default: true, aliases: '-f'
-  option :submit, desc: 'Submit results (if any) to MesaTestHub.',
+  option :submit, desc: 'Submit results (if any) to MESATestHub.',
                   type: :boolean, default: true
   option :module, desc: 'Which module to test and/or submit, if applicable.',
                   type: :string, default: 'all', aliases: '-m'
   option :destroy, type: :boolean, desc: 'Delete mesa installation upon ' \
                    'submission of complete results.', default: false
+  claim_option
+  run_mode_options
+  build_threads_option
+  parallel_options
 
   def install_and_test(sha='main')
+    validate_options!
+    return install_and_test_best if sha == 'best'
+
     s = if options[:submit]
           create_and_check_submitter(force: options[:force])
         else
@@ -290,38 +413,80 @@ class MesaTest < Thor
           )
         end
 
+    modes = explicit_run_modes
+    claiming = options[:submit] && options[:claim]
+    s.claim(sha: m.sha, scope: 'build', flags: claim_flags(modes)) if claiming
+
     begin
-      m.clean
-      m.install
+      build_with_modes(m, modes, raise_on_failure: true)
     rescue MesaDirError
-      shell.say "\nFailed in compiling MESA.", :red
-    else
-      check_mesa(m)
-      update_mesa(mesa: m, mod: options[:module].downcase.to_sym)
-
-      # run all tests. Don't be fooled, whether or not a diff happens is set
-      # by the diff attribute of the mesa object, set above.
-      m.with_mesa_dir { m.each_test_run(mod: options[:module].downcase.to_sym) }
-    ensure
-      # submit all commit and test data
-      successfully_submitted = false
-      if options[:submit]
-        successfully_submitted = s.submit_commit(
-          m, force_logs: options['force-logs'])
+      # Report the failed build (with its log) right away.
+      submit_build(s, m) if options[:submit]
+      exit 1
+    end
+
+    check_mesa(m)
+    update_mesa(mesa: m, mod: options[:module].downcase.to_sym)
+    # The build's status goes in as soon as it's known, then each test as
+    # soon as it finishes -- results roll in rather than arriving in one
+    # bulk submission at the end.
+    build_submitted = options[:submit] ? submit_build(s, m) : true
+    all_submitted = with_env(run_mode_env(m.recorded_run_modes).merge(test_thread_env)) do
+      m.with_mesa_dir do
+        run_tests(s, m, mod: options[:module].downcase.to_sym,
+                        submit: options[:submit], flags: claim_flags(modes))
       end
+    end
 
-      # if requested, installation and submission are successful, destroy the
-      # directory
-      m.remove if successfully_submitted && options[:destroy] && m.installed?
+    # if requested, installation and submission are successful, destroy the
+    # directory
+    if options[:submit] && build_submitted && all_submitted && options[:destroy] && m.installed?
+      m.remove
     end
   end
 
-  desc 'setup [CONFIG_FILE]', 'Setup MesaTestHub config file.'
+  desc 'request_work', 'ask MESATestHub what this computer should do next'
   long_desc <<-LONGDESC
-    If optional CONFIG_FILE is provided, search for that file and load it
-    to provide default values for configuration. If not provided, the default
-    file, ~/.mesa_test.yml, is used. The resulting config file is saved to
-    the same location it was read from.
+  Asks MESATestHub which commit this computer should build next (or, with
+  --scope=test --sha=SHA, which test to run next on a commit it built) and
+  prints the recommendation as JSON on STDOUT, including the run modes to
+  use and the reasons it was picked. The mode flags say what this computer
+  is willing to do (overriding `capabilities` in the config file). Prints
+  nothing and exits with status #{NOTHING_TO_DO_EXIT} when there is nothing
+  useful to do. Read-only: nothing is claimed. Needs an API key (see
+  `mesa_test help setup`).
+  LONGDESC
+
+  option :scope, desc: 'build or test', type: :string, default: 'build',
+                 enum: %w[build test]
+  option :sha, desc: 'For --scope=test: the commit you built', type: :string
+  run_mode_options
+
+  def request_work
+    s = create_submitter(force: true, hub: true)
+    rec = dispatch_or_exit(s, scope: options[:scope], sha: options[:sha])
+    exit NOTHING_TO_DO_EXIT unless rec
+
+    puts JSON.pretty_generate(rec)
+  end
+
+  desc 'setup [CONFIG_FILE]', 'Set up this computer (API key, test modes, directories)'
+  long_desc <<-LONGDESC
+    Walks through this computer's settings, offering the current ones as
+    defaults, and saves them to CONFIG_FILE (default ~/.mesa_test/config.yml,
+    which is where the other commands look).
+
+    mesa_test authenticates with a per-computer API key. To get one, log in
+    to MESATestHub, open My Computers, open this computer's page (adding the
+    computer first if needed; one used only for `search` and `count` is
+    fine), and click "Generate key". The key starts with mth_. Paste it when
+    the wizard asks, or export MESATESTHUB_API_KEY, which overrides the
+    config file (e.g. to keep the key off disk on a cluster).
+
+    The wizard checks the key with MESATestHub, which also supplies this
+    computer's name. Then it asks which optional test modes the computer is
+    willing to run when a commit asks for them (full_inlists, fpe, converge),
+    the logs token, and where the MESA mirror and work directory live.
   LONGDESC
   def setup(config_file = File.join(ENV['HOME'], '.mesa_test', 'config.yml'))
     MesaTestSubmitter.new_from_config(config_file: config_file,
@@ -330,9 +495,9 @@ class MesaTest < Thor
 
   desc 'search "QUERY"', 'retrieve JSON test instances matching QUERY'
   long_desc <<-LONGDESC
-  Sends a GET request to the MesaTestHub search API. QUERY should be
-  quoted. Credentials are read from your mesa_test config file and sent as
-  URL query parameters (HTTPS is mandatory).
+  Sends a GET request to the MESATestHub search API. QUERY should be
+  quoted. Authenticates with your API key; a key for any of your computers
+  works, since search is read-only.
 
   Query syntax is the same key-value form used by the in-browser search box
   at testhub.mesastar.org/test_instances/search (see the "Query syntax"
@@ -348,7 +513,7 @@ class MesaTest < Thor
   LONGDESC
 
   def search(query)
-    s = create_submitter(force: true)
+    s = create_submitter(force: true, hub: true)
     emit_search_response(s.search(query))
   end
 
@@ -360,12 +525,365 @@ class MesaTest < Thor
   LONGDESC
 
   def count(query)
-    s = create_submitter(force: true)
+    s = create_submitter(force: true, hub: true)
     emit_search_response(s.search_count(query))
   end
 
   private
 
+  # `install_and_test best`: MESATestHub picks the commit, then each test.
+  def install_and_test_best
+    unless options[:submit]
+      raise Thor::Error, '`install_and_test best` submits as it goes; ' \
+                         '--no-submit is not supported.'
+    end
+    s = create_and_check_submitter(force: options[:force])
+    rec = dispatch_build_or_exit(s)
+    modes = modes_for(rec)
+    m = Mesa.checkout(sha: rec['commit_sha'],
+                      work_dir: work_dir(options[:mesadir], s.mesa_work),
+                      mirror_dir: s.mesa_mirror,
+                      github_protocol: s.github_protocol)
+    if options[:claim]
+      s.claim(sha: m.sha, scope: 'build', flags: claim_flags(modes),
+              dispatched_at: rec['dispatched_at'])
+    end
+
+    begin
+      build_with_modes(m, modes, raise_on_failure: true)
+    rescue MesaDirError
+      submit_build(s, m)
+      exit 1
+    end
+    mod = options[:module].downcase.to_sym
+    check_mesa(m)
+    update_mesa(mesa: m, mod: mod)
+
+    # Report the build on its own; this also fulfills the build claim.
+    build_submitted = submit_build(s, m)
+    # MESATestHub chose the commit and its modes; from here it's an ordinary
+    # whole-suite run, as `install_and_test SHA` does it: MESA's order,
+    # --jobs at a time, each test claimed as it starts and submitted as it
+    # finishes.
+    all_submitted = with_env(run_mode_env(m.recorded_run_modes).merge(test_thread_env)) do
+      m.with_mesa_dir do
+        run_tests(s, m, mod: mod, submit: true, flags: claim_flags(modes))
+      end
+    end
+    m.remove if build_submitted && all_submitted && options[:destroy] && m.installed?
+  end
+
+
+  def jobs
+    j = options[:jobs].to_i
+    raise Thor::Error, '--jobs must be at least 1.' if j < 1
+
+    j
+  end
+
+  # OMP_NUM_THREADS for each test. --test-threads if given. Otherwise, with
+  # --jobs N, the shell's OMP_NUM_THREADS (else the core count) is treated
+  # as the budget for the whole run and split evenly across the N tests,
+  # rounded down, at least 1 -- a value set for serial runs shouldn't
+  # silently become N times as many threads. Serial runs leave the shell
+  # alone. Computed once, before anything changes ENV.
+  def test_thread_env
+    @test_thread_env ||= begin
+      n = options[:test_threads]
+      if n
+        raise Thor::Error, '--test-threads must be at least 1.' if n.to_i < 1
+
+        { 'OMP_NUM_THREADS' => n.to_i.to_s }
+      elsif jobs > 1
+        { 'OMP_NUM_THREADS' => split_threads.to_s }
+      else
+        {}
+      end
+    end
+  end
+
+  def split_threads
+    from_shell = ENV['OMP_NUM_THREADS'].to_i
+    budget, source = if from_shell.positive?
+                       [from_shell, "OMP_NUM_THREADS=#{from_shell}"]
+                     else
+                       [Etc.nprocessors, "#{Etc.nprocessors} cores"]
+                     end
+    each = [budget / jobs, 1].max
+    if budget < jobs
+      shell.say "--jobs #{jobs} is more than your #{source}; each test gets 1 thread " \
+                "(#{jobs} in all).", :yellow
+    else
+      idle = budget - each * jobs
+      @thread_note = "#{source} split #{jobs} ways#{"; #{idle} left idle" if idle.positive?}; " \
+                     '--test-threads sets it directly'
+    end
+    each
+  end
+
+  def build_thread_env
+    n = options[:build_threads]
+    return {} if n.nil?
+    raise Thor::Error, '--build-threads must be at least 1.' if n.to_i < 1
+
+    { 'NPROCS' => n.to_i.to_s, 'OMP_NUM_THREADS' => n.to_i.to_s }
+  end
+
+  # Submit the results +t+ wrote during the run that began at +started+.
+  # If it wrote none (each_test_run died early, the run was killed), there
+  # is nothing to report: an older testhub.yml must not be sent as this
+  # run's result. Its claim is left to lapse, which tells the hub the work
+  # was abandoned. Returns whether a result went through.
+  def submit_result(s, m, t, started)
+    return false unless t.ran_since?(started)
+
+    s.submit_instance(m, t, force_logs: options['force-logs'], quiet: true)
+  end
+
+  # Check numeric options before anything is checked out or built, so a
+  # typo fails in a second rather than after a long compile.
+  def validate_options!
+    { jobs: 1, test_threads: 1, build_threads: 1 }.each do |name, min|
+      value = options[name]
+      next if value.nil?
+      next if value.is_a?(Numeric) && value == value.to_i && value >= min
+
+      raise Thor::Error, "--#{name.to_s.tr('_', '-')} must be a whole number of at least #{min}."
+    end
+    explicit_factor
+    nil
+  end
+
+  # Report this build's status (and its log, if it failed). Returns whether
+  # the report went through; false if there was nothing to report.
+  def submit_build(s, m)
+    unless m.install_attempted?
+      shell.say 'No build to report (the install never got far enough to ' \
+                'write testhub.yml).', :red
+      return false
+    end
+    s.submit_commit(m, empty: true)
+  end
+
+  # The loaded tests of +mod+ (or every module), in do1_test_source order.
+  def tests_for(m, mod)
+    mods = mod == :all ? MesaTestCase.modules : [mod]
+    mods.flat_map do |md|
+      (m.test_case_names[md] || []).filter_map { |name| (m.test_cases[md] || {})[name] }
+    end
+  end
+
+  # Run every test in +mod+ (or all modules) on +m+'s build, in the current
+  # environment and MESA_DIR: one at a time (MESA's output on the terminal,
+  # as before), or --jobs at a time (output in each test's
+  # each_test_run.log). With +submit+, each test is claimed just as it
+  # starts and submitted as soon as it finishes, so a claim always means
+  # "running now" and results arrive continuously -- no bulk submission.
+  # Returns true if every submission went through.
+  def run_tests(s, m, mod:, submit:, flags: {})
+    m.check_installation
+    tests = tests_for(m, mod)
+    sha = m.sha
+    claiming = submit && options[:claim]
+    if jobs > 1
+      threads = test_thread_env['OMP_NUM_THREADS'] || ENV['OMP_NUM_THREADS']
+      say_step(shell, "Testing #{tests.size} tests, #{jobs} at a time" \
+                      "#{", #{threads} thread#{'s' unless threads == '1'} each" if threads}")
+      shell.say "(#{@thread_note})" if @thread_note
+      shell.say "Each test's output goes to each_test_run.log in its directory."
+    end
+    lock = Mutex.new
+    done = 0
+    all_submitted = true
+    run_one = lambda do |t|
+      if claiming
+        lock.synchronize do
+          s.claim(sha: sha, scope: 'test', mod: t.mod, test_case: t.test_name, flags: flags, mesa: m)
+        end
+      end
+      started = Time.now
+      if jobs > 1
+        t.run_isolated
+      else
+        say_step(shell, "[#{lock.synchronize { done } + 1}/#{tests.size}] Testing #{t.mod}/#{t.test_name}")
+        t.do_one
+      end
+      lock.synchronize do
+        done += 1
+        # (not `&&=` on the call: that would skip every later submission)
+        submitted = finish_test(s, m, t, started, submit: submit, prefix: "[#{done}/#{tests.size}] ")
+        all_submitted &&= submitted
+      end
+    end
+    if jobs > 1
+      parallel_each(tests, jobs) { |t| run_one.call(t) }
+    else
+      tests.each { |t| run_one.call(t) }
+    end
+    all_submitted
+  end
+
+  # Submit a finished test (with +submit+) and print its one progress line:
+  #   [12/107] star/1M_pre_ms_to_wd passed in 41.2 min, submitted
+  # Only the outcome is colored. Returns whether the result went in (true
+  # when not submitting).
+  def finish_test(s, m, t, started, submit:, prefix: '')
+    elapsed = format_duration(Time.now - started)
+    unless t.ran_since?(started)
+      line = "#{prefix}#{t.mod}/#{t.test_name} #{shell.set_color('wrote no results', :red)} " \
+             "after #{elapsed}"
+      shell.say(submit ? "#{line}; nothing submitted" : line)
+      return !submit
+    end
+
+    passed = begin
+      t.passed?
+    rescue StandardError
+      false
+    end
+    outcome = passed ? shell.set_color('passed', :green) : shell.set_color('FAILED', :red)
+    submitted = submit ? submit_result(s, m, t, started) : true
+    suffix = if !submit then ''
+             elsif submitted then ', submitted'
+             else ", #{shell.set_color('not submitted', :red)}"
+             end
+    shell.say "#{prefix}#{t.mod}/#{t.test_name} #{outcome} in #{elapsed}#{suffix}"
+    submitted
+  end
+
+  def format_duration(seconds)
+    if seconds < 90 then format('%d s', seconds.round)
+    elsif seconds < 2 * 3600 then format('%.1f min', seconds / 60.0)
+    else format('%.1f h', seconds / 3600.0)
+    end
+  end
+
+
+  # clean + install +m+ with +modes+ in the environment, then record the
+  # modes in $MESA_DIR/testhub.yml for later `test`/`submit` runs.
+  def build_with_modes(m, modes, raise_on_failure: false)
+    say_step(shell, "Building MESA #{short_sha(m.sha)}" \
+                    "#{" (#{describe_modes(modes)})" unless modes.empty?}")
+    with_env(run_mode_env(modes).merge(build_thread_env)) do
+      m.clean
+      m.install
+    end
+  rescue MesaDirError
+    shell.say "MESA #{short_sha(m.sha)} failed to build; see #{File.join(m.mesa_dir, 'build.log')}.", :red
+    raise if raise_on_failure
+  else
+    # Recorded so later `mesa_test test` runs on this build use the same modes.
+    m.record_run_modes(modes)
+    shell.say "Built MESA #{short_sha(m.sha)}.", :green
+  end
+
+  # Run modes given explicitly on the command line (unset flags omitted).
+  def explicit_run_modes
+    modes = {}
+    modes[:skip_optional] = options[:skip_optional] unless options[:skip_optional].nil?
+    modes[:fpe] = options[:fpe] unless options[:fpe].nil?
+    modes[:resolution_factor] = (options[:converge] ? converge_factor : false) unless options[:converge].nil?
+    modes[:resolution_factor] = explicit_factor if explicit_factor
+    modes
+  end
+
+  # --resolution-factor, as a string for the environment; implies
+  # --converge, so it can't be combined with --no-converge.
+  def explicit_factor
+    factor = options[:resolution_factor]
+    return nil if factor.nil?
+    raise Thor::Error, '--resolution-factor must be positive.' unless factor.positive?
+    raise Thor::Error, '--resolution-factor implies --converge; drop --no-converge.' if options[:converge] == false
+
+    factor.to_s
+  end
+
+  # The run modes a dispatch asks for, with any --resolution-factor given
+  # here as the preferred converge factor (a factor the commit names
+  # still wins, with a warning).
+  def modes_for(rec)
+    if explicit_factor
+      dispatch_run_modes(rec, preferred: explicit_factor, preferred_source: '--resolution-factor')
+    else
+      dispatch_run_modes(rec)
+    end
+  end
+
+  # Modes for running tests on +m+'s build: what was recorded at install,
+  # overridden by flags given now — except FPE, which is compiled in.
+  def test_run_modes(m)
+    recorded = m.downloaded? ? m.recorded_run_modes : {}
+    explicit = explicit_run_modes
+    if recorded.key?(:fpe) && explicit.key?(:fpe) && explicit[:fpe] != recorded[:fpe]
+      shell.say "Ignoring --#{explicit[:fpe] ? '' : 'no-'}fpe: this build was compiled " \
+                "#{recorded[:fpe] ? 'with' : 'without'} FPE checks.", :yellow
+      explicit.delete(:fpe)
+    end
+    # A factor the build recorded (e.g. from `[ci converge 0.5]` via
+    # `install best`) beats the shell's; say so when they differ.
+    shell_factor = ENV['MESA_TEST_SUITE_RESOLUTION_FACTOR']
+    if recorded[:resolution_factor] && !explicit.key?(:resolution_factor) && shell_factor &&
+       shell_factor.to_f != recorded[:resolution_factor].to_f
+      warn_yellow "This build was set up for resolution factor #{recorded[:resolution_factor]}; " \
+                  "using it instead of your MESA_TEST_SUITE_RESOLUTION_FACTOR=#{shell_factor}."
+    end
+    recorded.merge(explicit)
+  end
+
+  # What a claim says the work will run with (MESATestHub's use_* flags).
+  def claim_flags(modes)
+    {
+      fpe: modes[:fpe] == true,
+      full_inlists: modes[:skip_optional] == false,
+      converge: modes[:resolution_factor] ? true : false
+    }
+  end
+
+  def describe_modes(modes)
+    modes.map do |k, v|
+      case k
+      when :fpe then v ? 'FPE checks on' : 'FPE checks off'
+      when :skip_optional then v ? 'skip optional inlists' : 'run optional inlists'
+      when :resolution_factor then v ? "resolution factor #{v}" : 'default resolution'
+      end
+    end.join(', ')
+  end
+
+  # Optional modes this computer is willing to run for MESATestHub: the
+  # config file's `capabilities`, overridden by any mode flag given.
+  def capabilities_for(s)
+    caps = s.capabilities.dup
+    caps[:fpe] = options[:fpe] unless options[:fpe].nil?
+    caps[:full_inlists] = !options[:skip_optional] unless options[:skip_optional].nil?
+    caps[:converge] = options[:converge] unless options[:converge].nil?
+    caps[:converge] = true if options[:resolution_factor]
+    caps
+  end
+
+  # Ask for a commit to build, exiting with NOTHING_TO_DO_EXIT if there is
+  # none — before anything touches the work directory.
+  def dispatch_build_or_exit(s)
+    dispatch_or_exit(s, scope: 'build') || exit(NOTHING_TO_DO_EXIT)
+  end
+
+  # Dispatch, or print the server's complaint and exit 1. nil means there's
+  # nothing to do (said on STDERR so `request_work` STDOUT stays pure JSON).
+  def dispatch_or_exit(s, scope:, sha: nil, capabilities: capabilities_for(s))
+    rec = s.dispatch(scope: scope, sha: sha, capabilities: capabilities)
+    if rec.nil?
+      $stderr.puts 'MESATestHub has nothing for this computer to ' \
+                   "#{scope == 'build' ? 'build' : 'test on this commit'} right now."
+    elsif scope == 'build'
+      # (the hub's first reason is usually "on <branch>", already said)
+      reasons = Array(rec['reasons']) - ["on #{rec['branch']}"]
+      $stderr.puts "MESATestHub picked #{rec['short_sha']} on #{rec['branch']}: #{reasons.join('; ')}"
+    end
+    rec
+  rescue TestHubError => e
+    $stderr.puts e.message
+    exit 1
+  end
+
   def emit_search_response(response)
     if response.is_a?(Net::HTTPSuccess)
       puts response.body
@@ -374,6 +892,10 @@ class MesaTest < Thor
       $stderr.puts "Search request failed: HTTP #{response.code} "\
                    "#{response.message}"
       $stderr.puts response.body unless response.body.nil? || response.body.empty?
+      if response.code == '401'
+        $stderr.puts "\nCheck your API key (`mesa_test setup`, or MESATESTHUB_API_KEY).\n" \
+                     "#{MesaTestSubmitter::API_KEY_HELP}"
+      end
       exit 1
     end
   end
@@ -404,11 +926,16 @@ class MesaTest < Thor
       puts computer_check['message']
     else
       $stderr.puts computer_check['message']
+      unless computer_check['reachable'] == false
+        $stderr.puts 'Check your API key (`mesa_test setup`, or MESATESTHUB_API_KEY).'
+      end
       exit 1
     end
   end
 
-  def create_submitter(force: true)
+  # hub: true for commands that talk to MESATestHub, which all need an API
+  # key; checked first, before any checkout or build.
+  def create_submitter(force: true, hub: false)
     s = case MODE
         when :development
           MesaTestSubmitter.new_from_config(base_uri: DEVELOPMENT_URI)
@@ -420,11 +947,12 @@ class MesaTest < Thor
     unless force
       s.setup unless s.confirm_computer_data
     end
+    s.require_api_key! if hub
     s
   end
 
   def create_and_check_submitter(force: true)
-    s = create_submitter(force: force)
+    s = create_submitter(force: force, hub: true)
     check_user_and_computer s
     s
   end
@@ -467,4 +995,13 @@ class MesaTest < Thor
 end
 
 # actually start the CLI
-MesaTest.start(ARGV)
+begin
+  MesaTest.start(ARGV)
+rescue Interrupt
+  # run_workers has already stopped any tests running in parallel.
+  $stderr.puts "\nInterrupted."
+  exit 130
+rescue TestHubError => e
+  $stderr.puts e.message
+  exit 1
+end