mesa_test 0.2.15 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 258ab45be43e85b9d9bf5bb5ee31014f845ce52a760abc5d36d52efefec783fc
-  data.tar.gz: 179db21ba9f4da610bc0491e089eb8de5a8eb18d003682c7432579cdf80c4363
+  metadata.gz: cdee517026a8d0cca40ba51a9ad5418a5d5ac0e52977dcca9fdd9fcb32ead0b2
+  data.tar.gz: 7913ccaed137662943c1aa6ff06853c8641674c9fb822d5ee7391a2a12042ccd
 SHA512:
-  metadata.gz: faeacd323506e78ec1788191664aa411b18d844e2ee2e7b1ed7ad6e392415d6e8aa2aa73a9e124ca5e18dde8ec8536eb07b15c49a0bfeda3c6750b202f9604cc
-  data.tar.gz: e6658640ae43198ed869d1b991f185b030190ba1d42609e9ebc721b22bfafdda6dd861c4956b20ab64e84341577d38f72927dd25e73a64bc47cb265e9495ad11
+  metadata.gz: 6ffe69d78fb35653875ef3232e31c3f1cad46342281df3825a81ff364d113049658681140afadabfb7db2dcb85ea4a4046bc79117632b35fbc37f6b2dfc19251
+  data.tar.gz: e130047bbd95f236aec698240e0c9633d27185daf1c8cd4cdb8448773a675765187192386b81f91a7095ca6a71cbbe2ac3a30b6a3ff3669a3ab4b0efbfae5a46

data/bin/mesa_test

@@ -2,9 +2,10 @@
 
 # if true, submit cases to localhost:3000 and load local mesa_test library
 # rather than system version
-DEVELOPMENT_MODE = false
+MODE = :production
 
-if DEVELOPMENT_MODE
+case MODE
+when :staging, :development
   require_relative '../lib/mesa_test'
 else
   require 'mesa_test'
@@ -15,448 +16,302 @@ require 'yaml'
 
 class MesaTest < Thor
   DEVELOPMENT_URI = 'http://localhost:3000'.freeze
+  STAGING_URI = 'https://beta-testhub.herokuapp.com'.freeze
 
-  desc 'setup [CONFIG_FILE]', 'Setup MesaTestHub config file.'
+  desc 'test [TEST CASE]', 'run and submit a test case or commit'
   long_desc <<-LONGDESC
-    If optional CONFIG_FILE is provided, search for that file and load it
-    to provide default values for configuration. If not provided, the default
-    file, ~/.mesa_test.yml, is used. The resulting config file is saved to
-    the same location it was read from.
+    If you specify an optional test case, only that test case
+    will be tested. Otherwise all test cases in your work directory (set up in
+    your .mesa_test.yml file). TEST_CASE can also be an integer corresponding
+    to the position it has in do1_test_source (or a concatenated version of
+    that file if no module is specified). Modules are searched/concatenated in
+    this order:
+
+    #{MesaTestCase.modules.map { |mod| '    ' + mod.to_s }.join(', ')}
   LONGDESC
-  def setup(config_file = File.join(ENV['HOME'], '.mesa_test.yml'))
-    MesaTestSubmitter.new_from_config(config_file: config_file,
-                                      force_setup: true)
-  end
-
-  desc 'test_one MESA_DIR TEST_CASE', 'run, check, and submit one test case'
-  long_desc <<-LONGDESC
-  Run and check TEST_CASE, which resides in MESA_DIR/star/test_suite. Then
-  report results to MesaTestHub. TEST_CASE can also be an integer corresponding
-  to the line number of the test name in its do1_test_source file (or a
-  concatenated version of that file if no module is specified). Modules are
-  searched/concatenated in this order:
 
-  #{MesaTestCase.modules.map { |mod| '    ' + mod.to_s }.join(', ')}
-  LONGDESC
-  option :force, desc: 'When submitting, assume submission data in ' +
-                       '~/.mesa_test.yml are correct rather than checking ' +
-                       'with user.', type: :boolean, default: false,
-                       aliases: '-f'
+  option :force, desc: 'When submitting, assume submission data in ' \
+                       '~/.mesa_test.yml are correct rather than checking ' \
+                       'with user.',
+                 type: :boolean, default: true, aliases: '-f'
   option :submit, desc: 'Submit results (if any) to MesaTestHub.',
                   type: :boolean, default: true
-  option :diff, type: :boolean, default: false, desc: 'Assume checksums are ' + 
-                'up to date and perform diff', aliases: '-d'
-  option :svn, desc: 'Use svn to gather log/version data. Safe to use when ' +
-               'also using --no-submit; else may give bad version numbers.',
-               type: :boolean, default: true
-  option :set_sdk, desc: 'Override user- and test-generated data to deterine' +
-                   ' if the SDK is used and instead use value of --using_sdk.',
-                   type: :boolean, default: false
-  option :using_sdk, desc: 'Assume we are using the SDK and automatically ' +
-                     'determine SDK version. Ignored unless using --set-sdk).',
-                     type: :boolean, default: true
   option :module, desc: 'Which module to test and/or submit, if applicable.',
-                  type: :string, default: 'all', aliases: '-m'                  
-  option :log, type: :boolean, default: true, desc: 'Save yml file of test ' \
-               'results in test case directory.'
-  option :auto_diff, desc: 'Automatically determine if diffs should be run '\
-                     'from svn log entry. Overrides --diff or --no-diff.',
-                     type: :boolean, default: false
-  def test_one(mesa_dir, test_case_name)
-    s = create_and_check_submitter(force: options[:force]) if options[:submit]
-    m = create_and_check_mesa(mesa_dir: mesa_dir, diff: options[:diff],
-                              mod: options[:module].downcase.to_sym,
-                              use_svn: options[:svn], using_sdk: using_sdk?(s),
-                              auto_diff: options[:auto_diff])
-    t = create_and_check_test_case(mesa: m, test_case_name: test_case_name,
-                                   mod: options[:module].downcase.to_sym)
-
-    # clean and run test
-    t.clean
-    t.do_one
-
-    # log results
-    t.log_results if options[:log]
-
-    # bail out if not submitting
-    return unless options[:submit]
-
-    # submit results
-    print 'Submitting results to ' + s.base_uri + '... '
-    s.submit(t)
-    puts "Done.\n"
-  end
+                  type: :string, default: 'all', aliases: '-m'
+  option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
+                   type: :string
 
-  desc 'test_revision MESA_DIR', 'test and submit all tests as a packaged version'
-  long_desc <<-LONGDESC
-  Run and check all test cases residing in MESA_DIR/star/test_suite. Then
-  report results to MesaTestHub. Specifically, runs and checks all tests
-  detailed in MESA_DIR/star/test_suite/do1_test_source. Also submit version
-  information with test instances. Submits all test instances and version data
-  in a single request rather than case-by-case in `test_all`.
-  LONGDESC
+  def test(test_case_name = nil)
+    s = if options[:submit]
+          create_and_check_submitter(force: options[:force])
+        else
+          create_submitter(force: options[:force])
+        end
+    m = create_and_check_mesa(
+      mesa_dir: work_dir(options[:mesadir], s.mesa_work),
+      mirror_dir: s.mesa_mirror,
+      mod: options[:module].downcase.to_sym,
+      github_protocol: s.github_protocol
+    )
+    if test_case_name
+      # only testing one test case
+      t = create_and_check_test_case(mesa: m, test_case_name: test_case_name,
+                                     mod: options[:module].downcase.to_sym)
+
+      # run test
+      t.do_one
+
+      # bail out if not submitting
+      return unless options[:submit]
+
+      # submit results
+      shell.say 'Submitting results to ' + s.base_uri + '... ', :blue
+      s.submit_instance(m, t)
+      shell.say "Done.\n", :green
+    else
+      # run all tests
+      m.each_test_run(mod: options[:module].downcase.to_sym)
 
-  option :force, desc: 'When submitting, assume submission data in ' +
-                       '~/.mesa_test.yml are correct rather than checking ' +
-                       'with user.', type: :boolean, default: false,
-                       aliases: '-f'
-  option :submit, desc: 'Submit results (if any) to MesaTestHub.',
-                  type: :boolean, default: true
-  option :diff, type: :boolean, default: false, desc: 'Assume checksums are ' + 
-                'up to date and perform diff', aliases: '-d'
-  option :svn, desc: 'Use svn to gather log/version data. Safe to use when ' +
-               'also using --no-submit; else may give bad version numbers.',
-               type: :boolean, default: true
-  option :set_sdk, desc: 'Override user- and test-generated data to deterine' +
-                   ' if the SDK is used and instead use value of --using_sdk.',
-                   type: :boolean, default: false
-  option :using_sdk, desc: 'Assume we are using the SDK and automatically ' +
-                     'determine SDK version. Ignored unless using --set-sdk).',
-                     type: :boolean, default: true
-  option :module, desc: 'Which module to test and/or submit, if applicable.',
-                  type: :string, default: 'all', aliases: '-m'                  
-  option :log, type: :boolean, default: true, desc: 'Save yml file of test ' \
-               'results in test case directory.'
-  option :auto_diff, desc: 'Automatically determine if diffs should be run '\
-                     'from svn log entry. Overrides --diff or --no-diff.',
-                     type: :boolean, default: false
-  def test_revision(mesa_dir)
-    s = create_and_check_submitter(force: options[:force]) if options[:submit]
-    m = create_and_check_mesa(mesa_dir: mesa_dir, diff: options[:diff],
-                              mod: options[:module].downcase.to_sym,
-                              use_svn: options[:svn],
-                              using_sdk: using_sdk?(s),
-                              auto_diff: options[:auto_diff])
-
-    # run all tests
-    m.each_test_run_and_diff(log_results: options[:log],
-                             mod: options[:module].downcase.to_sym)
-
-    # submit all tests
-    s.submit_revision(m) if options[:submit]
-  end
+      # bail out if not submitting
+      return unless options[:submit]
 
+      shell.say 'Submitting results to ' + s.base_uri + '... ', :blue
+      s.submit_commit(m)
+      shell.say "Done.\n", :green
+    end
+  end
 
-  desc 'submit_one MESA_DIR TEST_CASE', 'submit one completed test case'
+  desc 'submit [TEST CASE]', 'submit a test case or commit'
   long_desc <<-LONGDESC
-  Load complete test case data from existing YAML file and submit to MESA.
-  TEST_CASE can also be an integer corresponding to the line number of the test
-  name in its do1_test_source file (or a concatenated version of that file if
-  no module is specified). Modules are searched/concatenated in this order:
-
-  #{MesaTestCase.modules.map { |mod| '    ' + mod.to_s }.join(', ')}
+    All tests are assumed to be in the default mesa directory (set as mesa_work
+    in .mesa_test.yml. If you specify an optional test case, only that test case
+    will be submitted. TEST_CASE can also be an integer corresponding to the
+    position it has in do1_test_source (or a concatenated version of that file
+    if no module is specified). Modules are searched/concatenated in this
+    order:
+
+    #{MesaTestCase.modules.map { |mod| '    ' + mod.to_s }.join(', ')}
   LONGDESC
 
-  option :force, desc: 'When submitting, assume submission data in ' +
-                       '~/.mesa_test.yml are correct rather than checking ' +
-                       'with user.', type: :boolean, default: false,
-                       aliases: '-f'
-  option :svn, desc: 'Use svn to gather log/version data. NECESSARY TO GET ' +
-               'ACCURATE VERSION NUMBER.', type: :boolean, default: true
-  option :set_sdk, desc: 'Override user- and test-generated data to deterine' +
-                   ' if the SDK is used and instead use value of --using_sdk.',
-                   type: :boolean, default: false
-  option :using_sdk, desc: 'Assume we are using the SDK and automatically ' +
-                     'determine SDK version. Ignored unless using --set-sdk).',
-                     type: :boolean, default: true
+  option :force, desc: 'When submitting, assume submission data in ' \
+                       '~/.mesa_test.yml are correct rather than checking ' \
+                       'with user.',
+                 type: :boolean, default: true, aliases: '-f'
   option :module, desc: 'Which module to test and/or submit, if applicable.',
                   type: :string, default: 'all', aliases: '-m'
-  def submit_one(mesa_dir, test_case_name)
+  option :empty, desc: 'Whether any test cases will be submitted. Useful for '\
+                       'indicating compilation status independently of ' \
+                       'single test submissions.',
+                 type: :boolean, default: false, aliases: '-e'
+  option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
+                   type: :string
+
+  def submit(test_case_name = nil)
     s = create_and_check_submitter(force: options[:force])
-    m = create_and_check_mesa(mesa_dir: mesa_dir,
-                              mod: options[:module].downcase.to_sym,
-                              using_sdk: using_sdk?(s), use_svn: options[:svn])
-    t = create_and_check_test_case(mesa: m, test_case_name: test_case_name,
-                                   mod: options[:module].downcase.to_sym)
-    # load test results
-    t.load_results
-
-    # submit results
-    print 'Submitting results to ' + s.base_uri + '... '
-    s.submit(t)
-    puts "Done.\n"
+    m = create_and_check_mesa(
+      mesa_dir: work_dir(options[:mesadir], s.mesa_work),
+      mirror_dir: s.mesa_mirror,
+      mod: options[:module].downcase.to_sym,
+      github_protocol: s.github_protocol
+    )
+    empty = options[:empty]
+
+    # default value for now; might change to false if we have a bad
+    # installation or if a test case is specified
+
+    if test_case_name
+      # just submitting one test case.
+      # this is not an empty submission since there is one test case (the
+      # +empty+ option is ignored in this case)
+
+      t = create_and_check_test_case(mesa: m, test_case_name: test_case_name,
+                                     mod: options[:module].downcase.to_sym)
+      # submit results
+      shell.say 'Submitting results to ' + s.base_uri + '... ', :blue
+      s.submit_instance(m, t)
+      shell.say "Done.\n", :green
+    else
+      # submitting compilation information and either all tests (entire) or
+      # no tests (empty)
+      begin
+        m.check_installation
+      rescue MesaDirError
+        shell.say %q{This MESA installation doesn't seem to be compiled } \
+          'properly. Submitting a compilation failure to MESATestHub.', :red
+        empty = true
+      ensure
+        # submit all tests
+        s.submit_commit(m, empty: empty)
+      end
+    end
   end
 
-  desc 'submit_revision MESA_DIR', 'submit version with pre-run test cases'
+  desc 'checkout SHA', 'checkout a commit'
   long_desc <<-LONGDESC
-  Load results for all test cases residing in MESA_DIR/star/test_suite. Then
-  submit results to MesaTestHub. Specifically, loads and submits all tests
-  detailed in MESA_DIR/star/test_suite/do1_test_source along with version data
-  (number, author, and commit message). Unlike submit_all, this submits all
-  test cases AND version info in a single request rather than just the cases
-  in series.
+  Checks out a particular commit to default mesa directory (mesa_work in 
+  .mesa_test.yml). Uses git worktree magic to avoid re-downloading all of
+  mesa history if a mirror repo is already set up. Does not affect the
+  user's MESA_DIR or other environment variables.
   LONGDESC
-  option :force, desc: 'When submitting, assume submission data in ' +
-                       '~/.mesa_test.yml are correct rather than checking ' +
-                       'with user.', type: :boolean, default: false,
-                       aliases: '-f'
-  option :module, desc: 'Which module to test and/or submit, if applicable.',
-                  type: :string, default: 'all', aliases: '-m'                       
-  option :svn, desc: 'Use svn to gather log/version data. NECESSARY TO GET ' +
-               'ACCURATE VERSION NUMBER.', type: :boolean, default: true
-  option :set_sdk, desc: 'Override user- and test-generated data to deterine' +
-                   ' if the SDK is used and instead use value of --using_sdk.',
-                   type: :boolean, default: false
-  option :using_sdk, desc: 'Assume we are using the SDK and automatically ' +
-                     'determine SDK version. Ignored unless using --set-sdk).',
-                     type: :boolean, default: true
-  def submit_revision(mesa_dir)
-    s = create_and_check_submitter(force: options[:force])
-    m = create_and_check_mesa(mesa_dir: mesa_dir,
-                              mod: options[:module].downcase.to_sym,
-                              use_svn: options[:svn], using_sdk: using_sdk?(s))
-    begin
-      m.check_installation
-    rescue MesaDirError
-      shell.say %q{This MESA installation doesn't seem to be compiled } \
-        'properly. Submitting a compilation failure to MESATestHub.', :red
-    else
-      # load all test data
-      m.each_test_load_results(mod: options[:module].downcase.to_sym)
-    ensure
-      # submit all tests
-      s.submit_revision(m)
-    end
-  end
 
+  option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
+                   type: :string
+
+  def checkout(sha)
+    # assuming config wizard has already been set (perhaps a little reckless)
+    # so that we can get at the work directory and mirror directory
+    s = create_submitter(force: true)
+    Mesa.checkout(sha: sha,
+                  work_dir: work_dir(options[:mesadir], s.mesa_work),
+                  mirror_dir: s.mesa_mirror,
+                  github_protocol: s.github_protocol)
+  end
 
-  desc 'install VERSION_NUMBER MESA_DIR', 'download and install mesa release '\
-    'VERSION_NUMBER to directory MESA_DIR'
+  desc 'install [SHA]', 'checkout and install a commit'
   long_desc <<-LONGDESC
-  Calls to svn to install mesa release VERSION_NUMBER into the directory
-  MESA_DIR. Basically just an svn checkout followed by going into the directory
-  and running ./clean and ./install. SDK or compilers must be set up prior.
-  Does not affect the user's MESA_DIR or other environment variables.
+  Optionally checks out a particular commit to default mesa directory 
+  (mesa_work in .mesa_test.yml). Then runs installation script in the work
+  directory. Equivalent to an [optional] git checkout followed by going
+  into the directory and running ./clean and ./install, but there is some git
+  worktree magic on the backend to improve efficiency. SDK or compilers must be
+  set up prior. Does not affect the user's MESA_DIR or other environment
+  variables outside of execution.
+
+  If no sha is provided, defaults to master (if checkout is occuring). SHA is
+  ignored if using option --no-checkout
   LONGDESC
-  def install(version, mesa_dir)
-    m = Mesa.download(version_number: version, new_mesa_dir: mesa_dir)
+
+  option :checkout, desc: 'Whether a checkout needs to happen. Use '\
+                          '--no-checkout if you already ran checkout.',
+                    type: :boolean, default: true, aliases: '-c'
+  option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
+                   type: :string
+
+  def install(sha = 'master')
+    # assuming config wizard has already been set (perhaps a little reckless)
+    # so that we can get at the work directory and mirror directory
+    s = create_submitter(force: true)
+    m = if options[:checkout]
+          Mesa.checkout(
+            sha: sha,
+            work_dir: work_dir(options[:mesadir], s.mesa_work),
+            mirror_dir: s.mesa_mirror,
+            github_protocol: s.github_protocol
+          )
+        else
+          create_and_check_mesa(
+            mesa_dir: work_dir(options[:mesadir], s.mesa_work),
+            mirror_dir: s.mesa_mirror,
+            mod: :all,
+            github_protocol: s.github_protocol
+          )
+        end
     m.clean
     m.install
   end
 
-  desc 'install_and_test_revision VERSION_NUMBER MESA_DIR', 'download and '\
-  'install mesa release VERSION_NUMBER to directory MESA_DIR, run all tests,'\
-  'and submit version and test data in single request.'
+  desc 'install_and_test [SHA]', 'Install, test, and submit an entire commit.'
   long_desc <<-LONGDESC
-  Calls to svn to install mesa release VERSION_NUMBER into the directory
-  MESA_DIR. Basically just an svn checkout followed by going into the directory
-  and running ./clean and ./install. SDK or compilers must be set up prior.
-  Once installation is complete, run the test suite, and report results
-  (version and all test data) to MesaTestHub in a single request (whereas
-  install_and_test_all only submits test data and does so in a serial fashion).
+  Optionally checks out a particular commit (specified by SHA, which default to
+  HEAD if not provided) to a mesa directory (mesa_work in .mesa_test.yml, 
+  unless a custom location is provided by --workdir=MYDIR). Then go into the
+  directory and run ./clean and ./install. SDK or compilers must be set up
+  prior. Once installation is complete, run the test suite, and report results
+  (commit and all test data) to MesaTestHub in a single request.
   Does not affect the user's MESA_DIR or other environment variables. This is
   basically a shortcut for running
 
-    `mesa_test install SOME_VERSION SOME_DIR`
+    `mesa_test install SOME_SHA`
 
   followed by
 
-    `mesa_test test_revision SOME_DIR`
-
-  If VERSION_NUMBER is 'non-paxton' or 'non_paxton', then the most recent
-  commit by anyone other than Bill Paxton ('bill_paxton') is tested and the
-  --diff flag is ignored and set to false. Note that using 'HEAD' for version
-  number will automatically select the most recent release, regardless of the
-  author.
+    `mesa_test test`
   LONGDESC
 
-  option :force, desc: 'When submitting, assume submission data in ' +
-                       '~/.mesa_test.yml are correct rather than checking ' +
-                       'with user.', type: :boolean, default: false,
-                       aliases: '-f'
+  option :checkout, desc: 'Whether a checkout needs to happen. Use '\
+                          '--no-checkout if you already ran checkout.',
+                    type: :boolean, default: true, aliases: '-c'
+  option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
+                   type: :string
+  option :force, desc: 'When submitting, assume submission data in ' \
+                       '~/.mesa_test.yml are correct rather than checking ' \
+                       'with user.',
+                 type: :boolean, default: true, aliases: '-f'
   option :submit, desc: 'Submit results (if any) to MesaTestHub.',
                   type: :boolean, default: true
-  option :diff, type: :boolean, default: false, desc: 'Assume checksums are ' + 
-                'up to date and perform diff', aliases: '-d'
-  option :svn, desc: 'Use svn to gather log/version data. Safe to use when ' +
-               'also using --no-submit; else may give bad version numbers.',
-               type: :boolean, default: true
-  option :set_sdk, desc: 'Override user- and test-generated data to deterine' +
-                   ' if the SDK is used and instead use value of --using_sdk.',
-                   type: :boolean, default: false
-  option :using_sdk, desc: 'Assume we are using the SDK and automatically ' +
-                     'determine SDK version. Ignored unless using --set-sdk).',
-                     type: :boolean, default: true
   option :module, desc: 'Which module to test and/or submit, if applicable.',
                   type: :string, default: 'all', aliases: '-m'
   option :destroy, type: :boolean, desc: 'Delete mesa installation upon ' \
                    'submission of complete results.', default: false
-  option :log, type: :boolean, default: true, desc: 'Save yml file of test ' \
-               'results in test case directory.'
-  option :auto_diff, desc: 'Automatically determine if diffs should be run '\
-                     'from svn log entry. Overrides --diff or --no-diff.',
-                     type: :boolean, default: false
-  def install_and_test_revision(version, mesa_dir)
-    s = nil # to make sure using_sdk? below will work properly
-    s = create_and_check_submitter(force: options[:force]) if options[:submit]
-    mesa_version, diff = filter_version_and_diff(version, s)
-    if mesa_version.nil?
-      puts 'No commits by anyone other than Bill Paxton since '\
-        "r#{s.last_tested}. Exiting."
-      return
-    end
-    m = Mesa.download(version_number: mesa_version, new_mesa_dir: mesa_dir,
-                      use_svn: options[:svn], using_sdk: using_sdk?(s))
+
+  def install_and_test(sha)
+    s = if options[:submit]
+          create_and_check_submitter(force: options[:force])
+        else
+          # not submitting? don't check config file (risky!) still need
+          # submitter to get work and mirror directories, though
+          create_submitter(force: true)
+        end
+
+    m = if options[:checkout]
+          Mesa.checkout(
+            sha: sha,
+            work_dir: work_dir(options[:mesadir], s.mesa_work),
+            mirror_dir: s.mesa_mirror,
+            github_protocol: s.github_protocol
+          )
+        else
+          create_and_check_mesa(
+            mesa_dir: work_dir(options[:mesadir], s.mesa_work),
+            mirror_dir: s.mesa_mirror,
+            mod: options[:module].to_sym,
+            github_protocol: s.github_protocol
+          )
+        end
+
     begin
       m.clean
       m.install
     rescue MesaDirError
-      shell.say "\nFailed in compiling MESA r-#{m.version_number}.", :red
+      shell.say "\nFailed in compiling MESA.", :red
     else
       check_mesa(m)
-      update_mesa(mesa: m, diff: diff, mod: options[:module].downcase.to_sym,
-                  auto_diff: options[:auto_diff])
+      update_mesa(mesa: m, mod: options[:module].downcase.to_sym)
 
       # run all tests. Don't be fooled, whether or not a diff happens is set
       # by the diff attribute of the mesa object, set above.
-      m.each_test_run_and_diff(mod: options[:module].downcase.to_sym,
-                               log_results: options[:log])
+      m.each_test_run(mod: options[:module].downcase.to_sym)
     ensure
-      # submit all version and test data
+      # submit all commit and test data
       successfully_submitted = false
-      successfully_submitted = s.submit_revision(m) if options[:submit]
+      successfully_submitted = s.submit_commit(m) if options[:submit]
 
       # if requested, installation and submission are successful, destroy the
       # directory
-      m.destroy if successfully_submitted && options[:destroy] && m.installed?
-    end
-  end
-
-  desc 'search "SEARCH_QUERY" [EMAIL] [PASSWORD]', 'retrieve JSON array of ' \
-    'test instances matching SEARCH_QUERY'
-  long_desc <<-LONGDESC
-  Sends a JSON request to the Mesa TestHub API. All of your SEARCH_QUERY should
-  be in quotes. For details on how to form a proper search query, visit 
-  testhub.mesastar.org/test_instances/search, where you can also test out
-  the search capability in a browser interface. Result is a single line of
-  JSON printed to STDOUT.
-
-  To prevent abuse, the API requires authentication with email and password.
-  By default, these are pulled from ~/.mesa_test.yml, but you may override
-  these with optional arguments.
-  LONGDESC
-
-  def search(search_query, email=nil, password=nil)
-    # get/confirm email and password
-    yml_file = File.join(ENV['HOME'], '.mesa_test.yml')
-    email, password = get_email_and_password(yml_file, email, password)
-    # prep API request
-    base_uri = if DEVELOPMENT_MODE
-                 'http://localhost:3000'
-               else
-                 'https://testhub.mesastar.org'
-               end
-    uri_search = URI.parse(base_uri + '/test_instances/search.json')
-    params = {
-      email: email,
-      password: password,
-      query_text: search_query
-    }
-    uri_search.query = URI.encode_www_form(params)
-
-    # perform search query
-    search_response = nil
-    Net::HTTP.start(uri_search.host, uri_search.port, use_ssl: base_uri.include?('https')) do |https|
-      request = Net::HTTP::Get.new(
-        uri_search,
-        initheader = { 'Content-Type' => 'application/json' }
-      )
-      search_response = JSON.parse(https.request(request).body)
-    end
-
-    # parse results
-    if search_response['error']
-      STDERR.puts "Error: #{search_response['error']}"
-    else
-      # break into sections
-      results = search_response["results"]
-      failures = search_response["failures"]
-
-      unless failures.empty?
-        STDERR.puts "The folloiwng options were invalid and ignored:"
-        failures.each { |failure| STDERR.puts "- #{failure}"}
-      end
-
-      shell.say results.to_json
+      m.remove if successfully_submitted && options[:destroy] && m.installed?
     end
   end
 
-  desc 'count "SEARCH_QUERY" [EMAIL] [PASSWORD]', 'count number of test ' \
-    'instances matching SEARCH_QUERY'
+  desc 'setup [CONFIG_FILE]', 'Setup MesaTestHub config file.'
   long_desc <<-LONGDESC
-  Sends a JSON request to the Mesa TestHub API. All of your SEARCH_QUERY should
-  be in quotes. For details on how to form a proper search query, visit 
-  testhub.mesastar.org/test_instances/search, where you can also test out
-  the search capability in a browser interface. Result is a single integer.
-
-  To prevent abuse, the API requires authentication with email and password.
-  By default, these are pulled from ~/.mesa_test.yml, but you may override
-  these with optional arguments.
+    If optional CONFIG_FILE is provided, search for that file and load it
+    to provide default values for configuration. If not provided, the default
+    file, ~/.mesa_test.yml, is used. The resulting config file is saved to
+    the same location it was read from.
   LONGDESC
-
-  def count(search_query, email=nil, password=nil)
-    # get/confirm email and password
-    yml_file = File.join(ENV['HOME'], '.mesa_test.yml')
-    email, password = get_email_and_password(yml_file, email, password)
-    # prep API request
-    base_uri = if DEVELOPMENT_MODE
-                 'http://localhost:3000'
-               else
-                 'https://testhub.mesastar.org'
-               end
-    uri_count = URI.parse(base_uri + '/test_instances/search_count.json')
-    params = {
-      email: email,
-      password: password,
-      query_text: search_query
-    }
-    uri_count.query = URI.encode_www_form(params)
-
-    # perform count query
-    count_response = nil
-    Net::HTTP.start(uri_count.host, uri_count.port, use_ssl: base_uri.include?('https')) do |https|
-      request = Net::HTTP::Get.new(
-        uri_count,
-        initheader = { 'Content-Type' => 'application/json' }
-      )
-      count_response = JSON.parse(https.request(request).body)
-    end
-
-    # parse results
-    if count_response['error']
-      STDERR.puts "Error: #{count_response['error']}"
-    else
-      # break into sections
-      count = count_response["count"]
-      failures = count_response["failures"]
-
-      unless failures.empty?
-        STDERR.puts "The folloiwng options were invalid and ignored:"
-        failures.each { |failure| STDERR.puts "- #{failure}"}
-      end
-
-      shell.say count, :blue
-    end
+  def setup(config_file = File.join(ENV['HOME'], '.mesa_test', 'config.yml'))
+    MesaTestSubmitter.new_from_config(config_file: config_file,
+                                      force_setup: true)
   end
 
   private
 
-  def get_email_and_password(file, email=nil, password=nil)
-    if email && password
-      return [email, password]
+  def work_dir(given, default)
+    if given
+      File.expand_path('', given)
+    else
+      default
     end
-    raise("File does not exist: #{file}.") unless File.exist?(file)
-    contents = File.read(file)
-    email_matcher = /^email\:\s*(?<email>.+@.+\..+)$/
-    password_matcher = /^password\:\s*(?<password>.+)$/
-
-    m = email_matcher.match(contents)
-    email ||= m[:email] if m
-
-    m = password_matcher.match(contents)
-    password ||= m[:password] if m
-
-    return [email, password]
   end
 
   def check_user_and_computer(submitter)
@@ -469,46 +324,52 @@ class MesaTest < Thor
     end
   end
 
-  def create_and_check_submitter(force: false)
-    s = if DEVELOPMENT_MODE
+  def create_submitter(force: true)
+    s = case MODE
+        when :development
           MesaTestSubmitter.new_from_config(base_uri: DEVELOPMENT_URI)
+        when :staging
+          MesaTestSubmitter.new_from_config(base_uri: STAGING_URI)
         else
           MesaTestSubmitter.new_from_config
         end
     unless force
       s.setup unless s.confirm_computer_data
     end
+    s
+  end
+
+  def create_and_check_submitter(force: true)
+    s = create_submitter(force: force)
     check_user_and_computer s
     s
   end
 
-  def create_and_check_mesa(mesa_dir:, use_svn: true, diff: true, mod: :all,
-                            using_sdk: true, auto_diff: false)
-    m = Mesa.new(mesa_dir: mesa_dir, use_svn: use_svn, using_sdk: using_sdk)
+  def create_and_check_mesa(mesa_dir: '', mirror_dir: '', mod: :all,
+                            github_protocol: nil)
+    m = Mesa.new(mesa_dir: mesa_dir, mirror_dir: mirror_dir,
+                 github_protocol: github_protocol)
     check_mesa(m)
-    update_mesa(mesa: m, diff: diff, mod: mod, auto_diff: auto_diff)
-    m
+    update_mesa(mesa: m, mod: mod)
   end
 
   def check_mesa(mesa)
-    unless mesa.downloaded?
-      raise MesaDirError, "Invalid MESA_DIR: #{mesa_dir}. Please download " \
-      'and install a valid MESA version or provide the path to one.'
-    end
+    return if mesa.downloaded?
+
+    raise MesaDirError, "Invalid MESA_DIR: #{mesa.mesa_dir}. Please "\
+      'download and install a valid MESA version or provide the path to one.'
   end
 
-  def update_mesa(mesa:, diff: true, mod: :all, auto_diff: false)
+  def update_mesa(mesa:, mod: :all)
     mesa.load_test_source_data(mod: mod)
-    # choose whether to update checksums.
-    mesa.update_checksums = !diff unless diff.nil?
-    # automatically figure out diff situation. Overrides choice from diff!
-    mesa.determine_diff if auto_diff
+    mesa
   end
 
   def create_and_check_test_case(mesa:, test_case_name:, mod: :all)
     t = mesa.find_test_case(test_case_name: test_case_name, mod: mod)
     # return test case if it was found
     return t unless t.nil?
+
     # test case was not found. Throw an error.
     msg = "No such test case, #{test_case_name} found in any of "
     # this should care about the value in `mod`. Current behavior is only
@@ -519,44 +380,6 @@ class MesaTest < Thor
     msg << '.'
     raise TestCaseDirError, msg
   end
-
-  def filter_version_and_diff(version, submitter)
-    if %w[non-paxton non_paxton].include?(version.to_s.downcase)
-      [Mesa.last_non_paxton_revision(submitter.last_tested), false]
-    else
-      [version, options[:diff]]
-    end
-  end
-
-  # determine whether or not we can assume the test should be / was run with
-  # the SDK. This results should only be relevant if a test is GOING TO BE RUN.
-  # If it has already been run, then the compiler and compiler version have
-  # already been set (or the opportunity was lost, either way this won't help.)
-  # 
-  # This gets used in a lot of places where it probably isn't necessary, just 
-  # to prevent problems later on if things change, as it does no harm when it
-  # doesn't help.
-  # 
-  # The only parameter is a submitter, if there is one to lean on. If there
-  # isn't, it's no problem.
-  def using_sdk?(s = nil)
-    # did user specify so? Trust them
-    if options[:set_sdk]
-      options[:using_sdk]
-    # are we submitting? scrape compiler info from that
-    elsif s
-      s.compiler == 'SDK'
-    # we don't have a submitter... need to read config file (same thing 
-    # submitter is doing)
-    elsif File.exist?(File.join(ENV['HOME'], '.mesa_test.yml'))
-      data = YAML.load(File.read(File.join(ENV['HOME'], '.mesa_test.yml')))
-      data['compiler'] == 'SDK'
-    # nothing works. Don't tell mesa that we are using SDK (we could still be
-    # using it, but it's up to a future submitter to tell the web site so.)
-    else
-      false
-    end
-  end
 end
 
 # actually start the CLI