mesa_test 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4400ba9260e7fb896e1ae980fb7a8795b5efeaec672fcea5ce108eed8ae8a355
-  data.tar.gz: 60c95a170bfbdbafa537dc99bf7dc23822ea6a5dbea1093ede0d46e6d7ae19aa
+  metadata.gz: 2c6b3c8632cb3efbbabbcbd9534f5519a317a0f80d991227dde9c7acc1257946
+  data.tar.gz: 81ac0670ba6f3e7799bd3f07ea571f9cee317a8cb2cc230fe8d7b2fa96837622
 SHA512:
-  metadata.gz: 938faed6625333b5890428d47495b0039b5a17bdaf09cdc703d38a4d07a1403c9f28d8a9fb98b360f9b9d5d32d48510d367b8c64381ef723ec58eb23550e72c7
-  data.tar.gz: b401ff50c34c01cda33f43fa5e3c5b88808f80ed6aaabd2300f2af15d7ac364dfb71f5880cf02ee9512f78e0967b34447ea98de304106eea2558bd0d8891e687
+  metadata.gz: f23b4cb696c153072d2890d4c6ec1c412ed6e9ce723568367ae731ca8b4189ada86dfcaca27d04899f14ec175971beafe2e15c91488ac777181200de8bf54f0a
+  data.tar.gz: 5d4c7559f31733ad12c88bec794663ac873b36fa2166129ceac9d17cb29ac8db019110f853360b0b60a6067a9ca469b3c26e35e1e44581b16be514df8297a544

data/bin/mesa_test

@@ -138,7 +138,8 @@ class MesaTest < Thor
         m.check_installation
       rescue MesaDirError
         shell.say %q{This MESA installation doesn't seem to be compiled } \
-          'properly. Submitting a compilation failure to MESATestHub.', :red
+          'properly. Attempting to submit a compilation failure to '\
+          'MESATestHub.', :red
         empty = true
       ensure
         # submit all tests

data/lib/mesa_test.rb

@@ -7,6 +7,7 @@ require 'net/http'
 require 'net/https'
 require 'thor'
 require 'json'
+require 'base64'
 
 MesaDirError = Class.new(StandardError)
 TestCaseDirError = Class.new(StandardError)
@@ -48,9 +49,14 @@ e-mail and password will be stored in plain text.'
         "#{s.email} (required)? (#{s.password})", :blue
       s.password = response unless response.empty?
 
+      # Get API key for submitting failure logs
+      response = shell.ask 'What is the logs submission API token associated '\
+        "with the email #{s.email} (required)? (#{s.logs_token})", :blue
+      s.logs_token = response unless response.empty?
+
       # Determine if we'll use ssh or https to access github
       response = shell.ask 'When accessing GitHub, which protocol do you '\
-        'want to use? ', :blue, limited_to: %w[ssh https]
+        'want to use?', :blue, limited_to: %w[ssh https]
       s.github_protocol = response.strip.downcase.to_sym
 
       # Get location of source MESA repo (the mirror)
@@ -75,10 +81,12 @@ e-mail and password will be stored in plain text.'
         "Ubuntu 16.04)? (#{s.platform_version}):", :blue
       s.platform_version = response unless response.empty?
 
-      # Confirm save location
-      response = shell.ask "This will be saved in #{s.config_file}. Press " \
-        'enter to accept or enter a new location:', :blue, path: true
-      s.config_file = response unless response.empty?
+      # we are powerless to do change the location for now, so stop asking
+      # about it
+      # # Confirm save location
+      # response = shell.ask "This will be saved in #{s.config_file}. Press " \
+      #   'enter to accept or enter a new location:', :blue, path: true
+      # s.config_file = response unless response.empty?
     end
 
     # Confirm data. If not confirmed, restart whole wizard.
@@ -108,7 +116,7 @@ e-mail and password will be stored in plain text.'
 
   attr_accessor :computer_name, :user_name, :email, :password, :platform,
                 :mesa_mirror, :mesa_work, :platform_version, :processor,
-                :config_file, :base_uri, :last_tested, :github_protocol
+                :config_file, :base_uri, :github_protocol, :logs_token
 
   attr_reader :shell
 
@@ -116,7 +124,7 @@ e-mail and password will be stored in plain text.'
   def initialize(
       computer_name: nil, user_name: nil, email: nil, github_protocol: nil,
       mesa_mirror: nil, platform: nil, platform_version: nil, processor: nil,
-      config_file: nil, base_uri: nil, last_tested: nil
+      config_file: nil, base_uri: nil, logs_token: nil
   )
     @computer_name = computer_name || Socket.gethostname.scan(/^[^\.]+\.?/)[0]
     @computer_name.chomp!('.') if @computer_name
@@ -144,6 +152,7 @@ e-mail and password will be stored in plain text.'
     @config_file = config_file || File.join(ENV['HOME'], '.mesa_test',
                                             'config.yml')
     @base_uri = base_uri
+    @logs_token = logs_token || ENV['MESA_LOGS_TOKEN']
 
     # set up thor-proof way to get responses from user. Thor hijacks the
     # gets command, so we have to use its built-in "ask" method, which is
@@ -164,11 +173,12 @@ e-mail and password will be stored in plain text.'
     puts "Computer Name           #{computer_name}"
     puts "User email              #{email}"
     puts 'Password                ***********'
+    puts "logs API token          #{logs_token}"
     puts "GitHub Protocol         #{github_protocol}"
     puts "MESA Mirror Location    #{mesa_mirror}"
     puts "MESA Work Location      #{mesa_work}"
     puts "Platform                #{platform} #{platform_version}"
-    puts "Config location         #{config_file}"
+    # puts "Config location         #{config_file}"
     puts '-------------------------------------------------------'
     puts ''
     response = shell.ask 'Is this correct? (y/Y = Yes, anything else = No):'
@@ -186,6 +196,7 @@ e-mail and password will be stored in plain text.'
       'computer_name' => computer_name,
       'email' => email,
       'password' => password,
+      'logs_token' => logs_token,
       'github_protocol' => github_protocol,
       'mesa_mirror' => mesa_mirror,
       'mesa_work' => mesa_work,
@@ -204,6 +215,7 @@ e-mail and password will be stored in plain text.'
     @computer_name = data_hash['computer_name']
     @email = data_hash['email']
     @password = data_hash['password']
+    @logs_token = data_hash['logs_token']
     @github_protocol = data_hash['github_protocol'].to_sym
     @mesa_mirror = data_hash['mesa_mirror']
     @mesa_work = data_hash['mesa_work']
@@ -271,6 +283,28 @@ e-mail and password will be stored in plain text.'
     res
   end
 
+  # Parameters for reporting a failed compilation to the logs server
+  def build_log_params(mesa)
+    {
+      'computer_name' => computer_name,
+      'commit' => mesa.sha,
+      'build.log' => mesa.build_log_64
+    }
+  end
+
+  # Parameters for reporting a failed test to the logs server
+  def test_log_params(test_case)
+    res = {
+      'computer_name' => computer_name,
+      'commit' => test_case.mesa.sha, 
+      'test_case' => test_case.test_name     
+    }
+    res['mk.txt'] = test_case.mk_64 unless test_case.mk_64.empty?
+    res['out.txt'] = test_case.out_64 unless test_case.out_64.empty?
+    res['err.txt'] = test_case.err_64 unless test_case.err_64.empty?
+    res
+  end
+
   # Parameters for a single test case. +mesa+ is an instance of +Mesa+, and
   # +test_case+ is an instance of MesaTestCase representing the test case to
   # be submitted
@@ -287,7 +321,7 @@ e-mail and password will be stored in plain text.'
     https.use_ssl = base_uri.include? 'https'
 
     request = Net::HTTP::Post.new(
-      uri, initheader = { 'Content-Type' => 'application/json' }
+      uri, initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' }
     )
     request.body = {
       email: email,
@@ -300,20 +334,26 @@ e-mail and password will be stored in plain text.'
   # submit entire commit's worth of test cases, OR submit compilation status
   # and NO test cases
   def submit_commit(mesa, empty: false)
+    unless mesa.install_attempted?
+      raise MesaDirError, 'No testhub.yml file found in installation; '\
+                          'must attempt to install before subitting.'
+    end
     uri = URI.parse(base_uri + '/submissions/create.json')
     https = Net::HTTP.new(uri.hostname, uri.port)
     https.use_ssl = true if base_uri.include? 'https'
 
     request = Net::HTTP::Post.new(
       uri,
-      initheader = { 'Content-Type' => 'application/json' }
+      initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' }
     )
 
     # create the request body for submission to the submissions API
     # 
     # if we have an empty submission, then it is necessarily not entire.
     # Similarly, a non-empty submission is necessarily entire (otherwise one
-    # would use +submit_instance+)
+    # would use +submit_instance+). Also, make a "nonempty" submission be
+    # empty if there was an overall build error
+    empty ||= !mesa.installed?
     request_data = {submitter: submitter_params,
                     commit: commit_params(mesa, empty: empty, entire: !empty)}
     # don't need test instances if it's an empty submission or if compilation
@@ -332,20 +372,47 @@ e-mail and password will be stored in plain text.'
       false
     else
       shell.say "\nSuccessfully submitted commit #{mesa.sha}.", :green
-      true
+      # commit submitted to testhub, now submit build log if compilation failed
+      # and exit
+      unless mesa.installed?
+        return submit_build_log(mesa)
+      end
+
+      # compilation succeded, so submit any logs for failing tests
+      res = true
+      unless empty
+        mesa.test_cases.each do |mod, test_case_hash|
+          test_case_hash.each do |tc_name, test_case|
+            # get at each individual test case, see if it failed, and if it
+            # did, submit its log files
+            unless test_case.passed?
+              res &&= submit_test_log(test_case)
+            end
+          end
+        end
+      end
+
+      # a true return value means that any and all log submission were
+      # successful
+      res
     end
   end
 
   # submit results for a single test case instance. Does *not* report overall
   # compilation status to testhub. Use an empty commit submission for that
   def submit_instance(mesa, test_case)
+    unless mesa.install_attempted?
+      raise MesaDirError, 'No testhub.yml file found in installation; '\
+                          'must attempt to install before subitting.'
+    end
+
     uri = URI.parse(base_uri + '/submissions/create.json')
     https = Net::HTTP.new(uri.hostname, uri.port)
     https.use_ssl = true if base_uri.include? 'https'
 
     request = Net::HTTP::Post.new(
       uri,
-      initheader = { 'Content-Type' => 'application/json' }
+      initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' }
     )
 
     # create the request body for submission to the submissions API
@@ -367,9 +434,80 @@ e-mail and password will be stored in plain text.'
     else
       shell.say "\nSuccessfully submitted instance of #{test_case.test_name} "\
                 "for commit #{mesa.sha}.", :green
+      # submit logs if test failed
+      return submit_test_log(test_case) unless test_case.passed?
+      true
+    end
+  end
+
+  # make generic request to LOGS server
+  # +params+ is a hash of data to be encoded as JSON and sent off
+  def submit_logs(params)
+    uri = URI('https://logs.mesastar.org/uploads')
+    https = Net::HTTP.new(uri.host, uri.port)
+    https.use_ssl = true
+    req = Net::HTTP::Post.new(uri.path, 'Content-Type' => 'application/json',
+                              'X-Api-Key' => logs_token)
+    req.body = params.to_json
+    https.request(req)
+  end
+
+  # send build log to the logs server
+  def submit_build_log(mesa)
+    # intercept and don't send if mesa was properly installed
+    return true if mesa.installed?
+
+    # don't even try unless we have a logs token set
+    unless logs_token
+      shell.say 'Cannot submit to logs server; need to set mesa_logs_token '\
+                'in the mesa_test config file.'
+      return false
+    end
+
+    # do submission
+    res = submit_logs(build_log_params(mesa))
+
+    # report out results
+    if !res.is_a? Net::HTTPOK
+      shell.say "\nFailed to submit build.log to the LOGS server for commit "\
+                "#{mesa.sha}.", :red
+      false
+    else
+      shell.say "\nSuccessfully submitted build.log to the LOGS server for "\
+                "#{mesa.sha}.", :green
       true
     end
   end
+
+  # send build log to the logs server
+  def submit_test_log(test_case)
+    # skip submission if mesa was never installed or if the test passed
+    return true if !test_case.mesa.installed? || test_case.passed?
+
+    # don't even try unless we have a logs token set
+    unless logs_token
+      shell.say 'Cannot submit to logs server; need to set mesa_logs_token '\
+                'in the mesa_test config file..'
+      return false
+    end
+  
+    # do submission
+    res = submit_logs(test_log_params(test_case))
+
+    # report out results
+    if !res.is_a? Net::HTTPOK
+      shell.say "Failed to submit out.txt and mk.txt to the LOGS server for "\
+                "test case #{test_case.test_name} in commit "\
+                "#{test_case.mesa.sha}.", :red
+      false
+    else
+      shell.say "Successfully submitted out.txt and mk.txt to the LOGS "\
+                "server for test case #{test_case.test_name} in commit "\
+                "#{test_case.mesa.sha}.", :green
+      true
+    end
+  end
+
 end
 
 class Mesa
@@ -551,18 +689,29 @@ class Mesa
                         'show a successful installation).'
   end
 
+  # base 64-encoded contents of build.log
+  def build_log_64
+    build_log = File.join(mesa_dir, 'build.log')
+    return '' unless File.exist?(build_log)
+
+    b64_file(build_log)
+  end
+
   # sourced from $MESA_DIR/testhub.yml, which should be created after
   # installation
   def compiler_hash
     data_file = File.join(mesa_dir, 'testhub.yml')
-    unless File.exist? data_file
-      raise(MesaDirError, "Could not find file testhub.yml in #{mesa_dir}.")
+    res = {
+            compiler: 'Unknown',
+            sdk_version: 'Unknown',
+            math_backend: 'Unknown'
+          }
+    if File.exist? data_file
+      res = res.merge(YAML.safe_load(File.read(data_file)) || {})
+      # currently version_number is reported, but we don't need that in Git land
+      res.delete('version_number') # returns the value, not the updated hash
+      res
     end
-
-    res = YAML.safe_load(File.read(data_file))
-    # currently version_number is reported, but we don't need that in Git land
-    res.delete('version_number') # returns the value, not the updated hash
-    res
   end
 
   ## TEST SUITE METHODS
@@ -600,12 +749,16 @@ class Mesa
           num, tc_name = line.strip.split
           @names_to_numbers[mod][tc_name.strip] = num.to_i
           @test_case_names[mod] << tc_name.strip
-          @test_cases[mod][tc_name.strip] = MesaTestCase.new(
-            test: tc_name.strip,
-            mod: mod,
-            position: num.to_i,
-            mesa: self
-          )
+          begin
+            @test_cases[mod][tc_name.strip] = MesaTestCase.new(
+              test: tc_name.strip,
+              mod: mod,
+              position: num.to_i,
+              mesa: self
+            )
+          rescue TestCaseDirError
+            shell.say "No such test case #{tc_name.strip}. Skipping loading it.", :red
+          end
         end
       end
     end
@@ -645,11 +798,16 @@ class Mesa
   def installed?
     # assume build log reflects installation status; does not account for
     # mucking with modules after the fact
-    downloaded? && File.read(File.join(mesa_dir, 'build.log')).include?(
+    build_log = File.join(mesa_dir, 'build.log')
+    downloaded? && File.exist?(build_log) && File.read(build_log).include?(
       'MESA installation was successful'
     )
   end
 
+  def install_attempted?
+    File.exist? File.join(mesa_dir, 'testhub.yml')
+  end
+
   private
 
   # verify that mesa_dir is valid by checking for existence of test_suite
@@ -834,6 +992,36 @@ class MesaTestCase
     YAML.safe_load(File.read(testhub_file), [Symbol])
   end
 
+  # whether or not a test case has passed; only has meaning
+  # if we can load the results hash, though
+  def passed?
+    results_hash['outcome'] == :pass
+  end
+
+  # Base-64 encoded contents of mk.txt file
+  def mk_64
+    mk_file = File.join(test_case_dir, 'mk.txt')
+    return '' unless File.exist?(mk_file)
+
+    b64_file(mk_file)
+  end
+
+  # Base-64 encoded contents of err.txt file
+  def err_64
+    err_file = File.join(test_case_dir, 'err.txt')
+    return '' unless File.exist?(err_file)
+
+    b64_file(err_file)
+  end
+
+  # Base-64 encoded contents of out.txt file
+  def out_64
+    out_file = File.join(test_case_dir, 'out.txt')
+    return '' unless File.exist?(out_file)
+
+    b64_file(out_file)
+  end
+
   private
 
   # cd into the test case directory, do something in a block, then cd back
@@ -932,3 +1120,8 @@ end
 def bashticks(command)
   `bash -c "#{command}"`.chomp
 end
+
+# encode the contents of a file as base-64
+def b64_file(filename)
+  Base64.encode64(File.open(filename).read)
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mesa_test
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - William Wolf
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-17 00:00:00.000000000 Z
+date: 2021-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -81,7 +81,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.3
 signing_key:
 specification_version: 4
 summary: Command line tool for running and reporting the MESA test suites.