modsvaskr 0.0.4 → 0.1.0

data/lib/modsvaskr/logger.rb

@@ -5,21 +5,40 @@ module Modsvaskr
 
     class << self
       attr_accessor :log_file
+      attr_accessor :stdout_io
     end
     @log_file = File.expand_path('Modsvaskr.log')
+    @stdout_io = $stdout
 
     # Log on screen and in log file
     #
     # Parameters::
     # * *msg* (String): Message to log
     def log(msg)
-      complete_msg = "[ #{Time.now.strftime('%F %T')} ] - #{msg}"
-      puts complete_msg
+      complete_msg = "[ #{Time.now.strftime('%F %T')} ] - [ #{self.class.name.split('::').last} ] - #{msg}"
+      Logger.stdout_io << "#{complete_msg}\n"
       File.open(Logger.log_file, 'a') do |f|
         f.puts complete_msg
       end
     end
 
+    # Display an output to the user.
+    # This is not a log.
+    #
+    # Parameters::
+    # * *msg* (String): Message to output
+    def out(msg)
+      Logger.stdout_io << "#{msg}\n"
+    end
+
+    # Wait for the user to enter a line and hit Enter
+    #
+    # Result::
+    # * String: The line entered by the user
+    def wait_for_user_enter
+      @config.no_prompt ? "\n" : $stdin.gets
+    end
+
   end
 
 end

data/lib/modsvaskr/run_cmd.rb

@@ -12,8 +12,8 @@ module Modsvaskr
     # * *args* (Array<String>): Additional arguments to give the command [default: []]
     def run_cmd(cmd, args: [])
       Dir.chdir cmd[:dir] do
-        cmd_line = "#{cmd[:exe]} #{((cmd.key?(:args) ? cmd[:args] : []) + args).join(' ')}"
-        raise "Unable to execute command line from \"#{dir}\": #{cmd_line}" unless system cmd_line
+        cmd_line = "\"#{cmd[:exe]}\" #{((cmd.key?(:args) ? cmd[:args] : []) + args).join(' ')}".strip
+        raise "Unable to execute command line from \"#{cmd[:dir]}\": #{cmd_line}" unless system cmd_line
       end
     end
 

data/lib/modsvaskr/tests_runner.rb

@@ -1,6 +1,8 @@
 require 'json'
 require 'time'
+require 'tmpdir'
 require 'modsvaskr/tests_suite'
+require 'modsvaskr/in_game_tests_runner'
 
 module Modsvaskr
 
@@ -12,17 +14,11 @@ module Modsvaskr
     # Default values are for a standard Skyrim SE installation.
     #
     # Parameters::
+    # * *config* (Config): Main configuration
     # * *game* (Game): Game for which we run tests
-    # * *tests_poll_secs* (Integer): Number of seconds to poll for the game run and tests changes [default: 5]
-    # * *timeout_frozen_tests_secs* (Integer): Time out (in seconds) between tests updates: if no test has been updated before this timeout, consider the game has crashed. [default: 300]
-    def initialize(
-      game,
-      tests_poll_secs: 5,
-      timeout_frozen_tests_secs: 300
-    )
+    def initialize(config, game)
+      @config = config
       @game = game
-      @tests_poll_secs = tests_poll_secs
-      @timeout_frozen_tests_secs = timeout_frozen_tests_secs
       # Parse tests suites
       @tests_suites = Hash[Dir.glob("#{__dir__}/tests_suites/*.rb").map do |tests_suite_file|
         tests_suite = File.basename(tests_suite_file, '.rb').to_sym
@@ -40,7 +36,7 @@ module Modsvaskr
     # Result::
     # * Array<Symbol>: List of tests suites
     def tests_suites
-      @tests_suites.keys
+      @tests_suites.keys.sort
     end
 
     # Return test names for a given tests suite
@@ -50,6 +46,7 @@ module Modsvaskr
     # Result::
     # * Array<String>: Test names of this suite
     def discover_tests_for(tests_suite)
+      log "Discover tests for #{tests_suite}"
       discovered_tests = @tests_suites[tests_suite].discover_tests
       # Complete our tests information
       complete_info = tests_info
@@ -108,115 +105,57 @@ module Modsvaskr
       # Test names (ordered) to be performed in game, per tests suite
       # Hash< Symbol, Array<String> >
       in_game_tests = {}
-      selected_tests.each do |tests_suite, suite_selected_tests|
+      selected_tests.each do |tests_suite, selected_tests|
         if @tests_suites[tests_suite].respond_to?(:run_test)
           # Simple synchronous tests
-          suite_selected_tests.each do |test_name|
+          selected_tests.each do |test_name|
             # Store statuses after each test just in case of crash
             set_statuses_for(tests_suite, [[test_name, @tests_suites[tests_suite].run_test(test_name)]])
           end
         end
-        if @tests_suites[tests_suite].respond_to?(:auto_tests_for)
-          # We run the tests from the game itself, using AutoTest mod.
-          in_game_tests[tests_suite] = suite_selected_tests
-        end
+        # We run the tests from the game itself.
+        in_game_tests[tests_suite] = selected_tests if @tests_suites[tests_suite].respond_to?(:in_game_tests_for)
       end
       unless in_game_tests.empty?
-        # Get the list of AutoTest we have to run and that we will monitor
-        in_game_auto_tests = in_game_tests.inject({}) do |merged_auto_tests, (tests_suite, suite_selected_tests)|
-          merged_auto_tests.merge(@tests_suites[tests_suite].auto_tests_for(suite_selected_tests)) do |auto_test_suite, auto_test_tests1, auto_test_tests2|
-            (auto_test_tests1 + auto_test_tests2).uniq
-          end
-        end
-        in_game_auto_tests.each do |auto_test_suite, auto_test_tests|
-          # Write the JSON file that contains the list of tests to run
-          File.write(
-            "#{@game.path}/Data/SKSE/Plugins/StorageUtilData/AutoTest_#{auto_test_suite}_Run.json",
-            JSON.pretty_generate(
-              'stringList' => {
-                'tests_to_run' => auto_test_tests
-              }
-            )
-          )
-          # Clear the AutoTest test statuses
-          File.unlink("#{@game.path}/Data/SKSE/Plugins/StorageUtilData/AutoTest_#{auto_test_suite}_Statuses.json") if File.exist?("#{@game.path}/Data/SKSE/Plugins/StorageUtilData/AutoTest_#{auto_test_suite}_Statuses.json")
-        end
-        auto_test_config_file = "#{@game.path}/Data/SKSE/Plugins/StorageUtilData/AutoTest_Config.json"
-        # Write the JSON file that contains the configuration of the AutoTest tests runner
-        File.write(
-          auto_test_config_file,
-          JSON.pretty_generate(
-            'string' => {
-              'on_start' => 'run',
-              'on_stop' => 'exit'
+        # Keep track of the mapping between tests suites and in-game tests, per in-game tests suite.
+        # Associated info is:
+        # * *tests_suite* (Symbol): The tests suite that has subscribed to the statuses of some in-game tests of the in-game tests suite.
+        # * *in_game_tests* (Array<String>): List of in-game tests that the tests suite is interested in.
+        # * *selected_tests* (Array<String>): List of selected tests for which in-game tests are useful.
+        # Hash< Symbol, Array< Hash< Symbol, Object > > >
+        in_game_tests_subscriptions = {}
+        # List of all in-game tests to perform, per in-game tests suite
+        # Hash< Symbol, Array< String > >
+        merged_in_game_tests = {}
+        # Get the list of in-game tests we have to run and that we will monitor
+        in_game_tests.each do |tests_suite, selected_tests|
+          in_game_tests_to_subscribe = @tests_suites[tests_suite].in_game_tests_for(selected_tests)
+          in_game_tests_to_subscribe.each do |in_game_tests_suite, selected_in_game_tests|
+            in_game_tests_subscriptions[in_game_tests_suite] = [] unless in_game_tests_subscriptions.key?(in_game_tests_suite)
+            in_game_tests_subscriptions[in_game_tests_suite] << {
+              tests_suite: tests_suite,
+              in_game_tests: selected_in_game_tests,
+              selected_tests: selected_tests
             }
-          )
-        )
-        puts ''
-        puts '=========================================='
-        puts '= In-game tests are about to be launched ='
-        puts '=========================================='
-        puts ''
-        puts 'Here is what you need to do once the game will be launched (don\'t launch it by yourself, the test framework will launch it for you):'
-        puts '* Load the game save you want to test (or start a new game).'
-        puts ''
-        puts 'This will execute all in-game tests automatically.'
-        puts ''
-        puts 'It is possible that the game crashes during tests:'
-        puts '* That\'s a normal situation, as tests don\'t mimick a realistic gaming experience, and the Bethesda engine is not meant to be stressed like that.'
-        puts '* In case of game crash (CTD), the Modsvaskr test framework will relaunch it automatically and resume testing from when it crashed.'
-        puts '* In case of repeated CTD on the same test, the Modsvaskr test framework will detect it and skip the crashing test automatically.'
-        puts '* In case of a game freeze without CTD, the Modsvaskr test framework will detect it after a few minutes and automatically kill the game before re-launching it to resume testing.'
-        puts ''
-        puts 'If you want to interrupt in-game testing: invoke the console with ~ key and type stop_tests followed by Enter.'
-        puts ''
-        puts 'Press enter to start in-game testing (this will lauch your game automatically)...'
-        $stdin.gets
-        idx_launch = 0
-        old_statuses = auto_test_statuses
-        loop do
-          launch_game(use_autoload: idx_launch > 0)
-          idx_launch += 1
-          monitor_game_tests
-          # Check tests again
-          new_statuses = auto_test_statuses
-          # Update the tests results based on what has been run in-game
-          in_game_tests.each do |tests_suite, suite_selected_tests|
-            @tests_suites[tests_suite].set_statuses(
-              @tests_suites[tests_suite].
-                parse_auto_tests_statuses_for(suite_selected_tests, new_statuses).
-                select { |(test_name, _test_status)| suite_selected_tests.include?(test_name) }
-            )
-          end
-          log 'Test statuses after game run:'
-          print_test_statuses(new_statuses)
-          # Careful as this JSON file can be written by Papyrus that treat strings as case insensitive.
-          # cf. https://github.com/xanderdunn/skaar/wiki/Common-Tasks
-          auto_test_config = Hash[JSON.parse(File.read(auto_test_config_file))['string'].map { |key, value| [key.downcase, value.downcase] }]
-          if auto_test_config.dig('stopped_by') == 'user'
-            log 'Tests have been stopped by user. Stop looping.'
-            break
-          end
-          if auto_test_config.dig('tests_execution') == 'end'
-            log 'Tests have finished running. Stop looping.'
-            break
-          end
-          if new_statuses == old_statuses
-            log 'No changes in AutoTest tests statuses in the game run. Stop looping.'
-            break
+            merged_in_game_tests[in_game_tests_suite] = [] unless merged_in_game_tests.key?(in_game_tests_suite)
+            merged_in_game_tests[in_game_tests_suite] = (merged_in_game_tests[in_game_tests_suite] + selected_in_game_tests).uniq
           end
-          old_statuses = new_statuses
-          # We will start again. Leave some time to interrupt if we want.
-          log 'We are going to start again in 10 seconds. Press Enter now to interrupt it.'
-          key_pressed =
-            begin
-              Timeout.timeout(10) { $stdin.gets }
-            rescue
-              nil
+        end
+        in_game_tests_runner = InGameTestsRunner.new(@config, @game)
+        in_game_tests_runner.run(merged_in_game_tests) do |in_game_tests_suite, in_game_tests_statuses|
+          # This is a callback called for each in-game test status change.
+          # Update the tests results based on what has been run in-game.
+          # Find all tests suites that are subscribed to those in-game tests.
+          in_game_tests_subscriptions[in_game_tests_suite].each do |tests_suite_subscription|
+            selected_in_game_tests_statuses = in_game_tests_statuses.slice(*tests_suite_subscription[:in_game_tests])
+            unless selected_in_game_tests_statuses.empty?
+              tests_suite = @tests_suites[tests_suite_subscription[:tests_suite]]
+              tests_suite.set_statuses(
+                tests_suite.
+                  parse_auto_tests_statuses_for(tests_suite_subscription[:selected_tests], { in_game_tests_suite => selected_in_game_tests_statuses }).
+                  select { |(test_name, _test_status)| tests_suite_subscription[:selected_tests].include?(test_name) }
+              )
             end
-          if key_pressed
-            log 'Run interrupted by user.'
-            break
           end
         end
       end
@@ -257,174 +196,6 @@ module Modsvaskr
       @tests_info_cache = tests_info
     end
 
-    # Launch the game, and wait for launch to be successful (get a PID)
-    #
-    # Parameters::
-    # * *use_autoload* (Boolean): If true, then launch the game using AutoLoad
-    def launch_game(use_autoload:)
-      # Launch the game
-      @idx_launch = 0 unless defined?(@idx_launch)
-      Dir.chdir(@game.path) do
-        if use_autoload
-          log "Launch Game (##{@idx_launch}) using AutoLoad..."
-          system "\"#{@game.path}/Data/AutoLoad.cmd\" auto_test"
-        else
-          log "Launch Game (##{@idx_launch}) using configured launcher..."
-          system "\"#{@game.path}/#{@game.launch_exe}\""
-        end
-      end
-      @idx_launch += 1
-      # The game launches asynchronously, so just wait a little bit and check for the process
-      sleep 10
-      tasklist_stdout = nil
-      loop do
-        tasklist_stdout = `tasklist | find "#{@game.running_exe}"`.strip
-        break unless tasklist_stdout.empty?
-        log "#{@game.running_exe} is not running. Wait for its startup..."
-        sleep 1
-      end
-      @game_pid = Integer(tasklist_stdout.split(' ')[1])
-      log "#{@game.running_exe} has started with PID #{@game_pid}"
-    end
-
-    # Kill the game, and wait till it is killed
-    def kill_game
-      first_time = true
-      loop do
-        system "taskkill #{first_time ? '' : '/F'} /pid #{@game_pid}"
-        first_time = false
-        sleep 1
-        tasklist_stdout = `tasklist | find "#{@game.running_exe}"`.strip
-        break if tasklist_stdout.empty?
-        log "#{@game.running_exe} is still running. Wait for its kill..."
-        sleep 5
-      end
-    end
-
-    # Get the list of AutoTest statuses
-    #
-    # Result::
-    # * Hash<String, Array<[String, String]> >: Ordered list of AutoTest [test name, test status], per AutoTest tests suite
-    def auto_test_statuses
-      statuses = {}
-      `dir "#{@game.path}/Data/SKSE/Plugins/StorageUtilData" /B`.split("\n").each do |file|
-        if file =~ /^AutoTest_(.+)_Statuses\.json$/
-          auto_test_suite = $1
-          # Careful as this JSON file can be written by Papyrus that treat strings as case insensitive.
-          # cf. https://github.com/xanderdunn/skaar/wiki/Common-Tasks
-          statuses[auto_test_suite] = JSON.parse(File.read("#{@game.path}/Data/SKSE/Plugins/StorageUtilData/AutoTest_#{auto_test_suite}_Statuses.json"))['string'].map do |test_name, test_status|
-            [test_name.downcase, test_status.downcase]
-          end
-        end
-      end
-      statuses
-    end
-
-    # Loop while monitoring game tests progression.
-    # If the game exits, then exit also the loop.
-    # In case no test is being updated for a long period of time, kill the game (we consider it was frozen)
-    # Prerequisite: launch_game has to be called before.
-    def monitor_game_tests
-      log 'Start monitoring game testing...'
-      last_time_tests_changed = Time.now
-      # Get a picture of current tests
-      current_statuses = auto_test_statuses
-      loop do
-        still_running = true
-        begin
-          # Process.kill does not work when the game has crashed (the process is still detected as zombie)
-          # still_running = Process.kill(0, @game_pid) == 1
-          tasklist_stdout = `tasklist 2>&1`
-          still_running = tasklist_stdout.split("\n").any? { |line| line =~ /#{Regexp.escape(@game.running_exe)}/ }
-          # log "Tasklist returned no Skyrim:\n#{tasklist_stdout}" unless still_running
-        rescue Errno::ESRCH
-          log "Got error while waiting for #{@game.running_exe} PID: #{$!}"
-          still_running = false
-        end
-        # Log a diff in tests
-        new_statuses = auto_test_statuses
-        diff_statuses = diff_statuses(current_statuses, new_statuses)
-        unless diff_statuses.empty?
-          # Tests have progressed
-          last_time_tests_changed = Time.now
-          log '===== Test statuses changes:'
-          diff_statuses.each do |tests_suite, tests_statuses|
-            log "* #{tests_suite}:"
-            tests_statuses.each do |(test_name, test_status)|
-              log "  * #{test_name}: #{test_status}"
-            end
-          end
-        end
-        break unless still_running
-        # If the tests haven't changed for too long, consider the game has frozen, but not crashed. So kill it.
-        if Time.now - last_time_tests_changed > @timeout_frozen_tests_secs
-          log "Last time test have changed is #{last_time_tests_changed.strftime('%F %T')}. Consider the game is frozen, so kill it."
-          kill_game
-          break
-        end
-        current_statuses = new_statuses
-        sleep @tests_poll_secs
-      end
-      log 'End monitoring game testing.'
-    end
-
-    # Diff between test statuses.
-    #
-    # Parameters::
-    # * *statuses1* (Hash<String, Array<[String, String]> >): Test statuses, per test name (in a sorted list), per tests suite
-    # * *statuses2* (Hash<String, Array<[String, String]> >): Test statuses, per test name (in a sorted list), per tests suite
-    # Result::
-    # * Hash<String, Array<[String, String]> >: statuses2 - statuses1
-    def diff_statuses(statuses1, statuses2)
-      statuses = {}
-      statuses1.each do |tests_suite, tests_info|
-        if statuses2.key?(tests_suite)
-          # Handle Hashes as it will be faster
-          statuses1_for_test = Hash[tests_info]
-          statuses2_for_test = Hash[statuses2[tests_suite]]
-          statuses1_for_test.each do |test_name, status1|
-            if statuses2_for_test.key?(test_name)
-              if statuses2_for_test[test_name] != status1
-                # Change in status
-                statuses[tests_suite] = [] unless statuses.key?(tests_suite)
-                statuses[tests_suite] << [test_name, statuses2_for_test[test_name]]
-              end
-            else
-              # This test has been removed
-              statuses[tests_suite] = [] unless statuses.key?(tests_suite)
-              statuses[tests_suite] << [test_name, 'Deleted']
-            end
-          end
-          statuses2_for_test.each do |test_name, status2|
-            unless statuses1_for_test.key?(test_name)
-              # This test has been added
-              statuses[tests_suite] = [] unless statuses.key?(tests_suite)
-              statuses[tests_suite] << [test_name, status2]
-            end
-          end
-        else
-          # All test statuses have been removed
-          statuses[tests_suite] = tests_info.map { |(test_name, _test_status)| [test_name, 'Deleted'] }
-        end
-      end
-      statuses2.each do |tests_suite, tests_info|
-        # All test statuses have been added
-        statuses[tests_suite] = tests_info unless statuses1.key?(tests_suite)
-      end
-      statuses
-    end
-
-    # Print test statuses
-    #
-    # Parameters::
-    # * *statuses* (Hash<String, Array<[String, String]> >): Test statuses, per test name (in a sorted list), per tests suite
-    def print_test_statuses(statuses)
-      statuses.each do |tests_suite, statuses_for_type|
-        next_test_name, _next_test_status = statuses_for_type.find { |(_name, status)| status != 'ok' }
-        log "[ #{tests_suite} ] - #{statuses_for_type.select { |(_name, status)| status == 'ok' }.size} / #{statuses_for_type.size} - Next test to perform: #{next_test_name.nil? ? 'None' : next_test_name}"
-      end
-    end
-
   end
 
 end

data/lib/modsvaskr/tests_suites/exterior_cell.rb

@@ -98,16 +98,15 @@ module Modsvaskr
         tests
       end
 
-      # Get the list of tests to be run in the AutoTest mod for a given list of test names.
-      # AutoTest names are case insensitive.
+      # Get the list of tests to be run in-game for a given list of test names.
       # [API] - This method is mandatory for tests needing to be run in-game.
       #
       # Parameters::
       # * *tests* (Array<String>): List of test names
       # Result::
-      # * Hash<String, Array<String> >: List of AutoTest mod test names, per AutoTest mod tests suite
-      def auto_tests_for(tests)
-        { 'Locations' => tests }
+      # * Hash<Symbol, Array<String> >: List of in-game test names, per in-game tests suite
+      def in_game_tests_for(tests)
+        { locations: tests }
       end
 
       # Set statuses based on the result of AutoTest statuses.
@@ -116,11 +115,11 @@ module Modsvaskr
       #
       # Parameters::
       # * *tests* (Array<String>): List of test names
-      # * *auto_test_statuses* (Hash<String, Array<[String, String]> >): Ordered list of AutoTest [test name, test status], per AutoTest tests suite
+      # * *auto_test_statuses* (Hash<Symbol, Array<[String, String]> >): Ordered list of AutoTest [test name, test status], per AutoTest tests suite
       # Result::
       # * Array<[String, String]>: Corresponding list of [test name, test status]
       def parse_auto_tests_statuses_for(tests, auto_test_statuses)
-        auto_test_statuses.key?('Locations') ? auto_test_statuses['Locations'] : []
+        auto_test_statuses.key?(:locations) ? auto_test_statuses[:locations] : []
       end
 
     end