modsvaskr 0.0.4 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9342c8943f1a2deb5b824f349319dae072d8bfa69cbc1e68af95fc518805f35b
-  data.tar.gz: 7cfcdb9b25f93b670e516bf8e992fae6bd51249981df565be2024429c48a2957
+  metadata.gz: cf22182b9e2f38b08900947b107bac303c8d9c6eebbc7b10ce80b88a899fd4da
+  data.tar.gz: 58a6a9f16573766eec853391ef1aa0efd2a649a691e530a16bbc8de0268c11b9
 SHA512:
-  metadata.gz: 104e7a71646fecf55debb31fff6e7a7498b5a2ce263c3a6e1fbafe21fed84f54c09134376bfc9534b20d52f73db262ed0a2c42a3d03a747cad1f1f36b6bd62eb
-  data.tar.gz: 867b8bee8c3efe11380693cbfebc17c869094d344a3ba6e0aa9681807b298a00d6c5981a85fa75e4fa8b118dc4d687a4d24b8161f0ac640486eb2d9c92bff593
+  metadata.gz: 87849b6c50f3cc08db5d150fada9511ff412ab2458310a2f6e64c6126f7037e7e656a379c383b7598449e728be0f925bdb06df39a2e760895c4c9b3a1aca85ff
+  data.tar.gz: 10edfe86882d700c6a052276810b9e21aff2022a89e6b4d082ca03ab4eedb00a8ac0604e93a637492778823c7720133103c72d5adeb06d1240ada02d58a75fb5

data/lib/modsvaskr/config.rb

@@ -12,7 +12,19 @@ module Modsvaskr
     # Parameters::
     # * *file* (String): File containing configuration
     def initialize(file)
-      @config = YAML.load(File.read(file))
+      @config = YAML.load(File.read(file)) || {}
+      # Parse all game types plugins
+      # Hash<Symbol, Class>
+      @game_types = Hash[
+        Dir.glob("#{__dir__}/games/*.rb").map do |game_type_file|
+          require game_type_file
+          base_name = File.basename(game_type_file, '.rb')
+          [
+            base_name.to_sym,
+            Games.const_get(base_name.split('_').collect(&:capitalize).join.to_sym)
+          ]
+        end
+      ]
     end
 
     # Get the games list
@@ -21,9 +33,10 @@ module Modsvaskr
     # * Array<Game>: List of games
     def games
       unless defined?(@games)
-        @games = @config['games'].map do |game_info|
-          require "#{__dir__}/games/#{game_info['type']}.rb"
-          Games.const_get(game_info['type'].to_s.split('_').collect(&:capitalize).join.to_sym).new(self, game_info)
+        @games = (@config['games'] || []).map do |game_info|
+          game_type = game_info['type'].to_sym
+          raise "Unknown game type: #{game_type}. Available ones are #{@game_types.keys.join(', ')}" unless @game_types.key?(game_type)
+          @game_types[game_type].new(self, game_info)
         end
       end
       @games
@@ -37,6 +50,30 @@ module Modsvaskr
       @config['xedit']
     end
 
+    # Return the 7-Zip path
+    #
+    # Result::
+    # * String: The 7-Zip path
+    def seven_zip_path
+      @config['7zip']
+    end
+
+    # Return the automated keys to apply
+    #
+    # Result::
+    # * Array<String>: The list of automated keys
+    def auto_keys
+      @config['auto_keys'] || []
+    end
+
+    # Return the no_prompt flag
+    #
+    # Result::
+    # * Boolean: no_prompt flag
+    def no_prompt
+      @config['no_prompt'] || false
+    end
+
   end
 
 end

data/lib/modsvaskr/game.rb

@@ -16,10 +16,22 @@ module Modsvaskr
     # * *game_info* (Hash<String,Object>): Game info:
     #   * *name* (String): Game name
     #   * *path* (String): Game installation dir
+    #   * *launch_exe* (String): Executable to be launched
+    #   * *min_launch_time_secs* (Integer): Minimum expected lauch time for the game, in seconds [default: 10]
+    #   * *tests_poll_secs* (Integer): Interval in seconds to be respected between 2 test statuses polling [default: 5]
+    #   * *timeout_frozen_tests_secs* (Integer): Timeout in seconds of a frozen game [default: 300]
+    #   * *timeout_interrupt_tests_secs* (Integer): Timeout in seconds for the player to interrupt a tests session before restarting the game [default: 10]
     def initialize(config, game_info)
       @config = config
-      @game_info = game_info
+      # Set default values here
+      @game_info = {
+        'min_launch_time_secs' => 10,
+        'tests_poll_secs' => 5,
+        'timeout_frozen_tests_secs' => 300,
+        'timeout_interrupt_tests_secs' => 10
+      }.merge(game_info)
       @name = name
+      @pid = nil
       init if self.respond_to?(:init)
     end
 
@@ -47,6 +59,30 @@ module Modsvaskr
       @game_info['launch_exe']
     end
 
+    # Return the tests polling interval
+    #
+    # Result::
+    # * Integer: Tests polling interval
+    def tests_poll_secs
+      @game_info['tests_poll_secs']
+    end
+
+    # Return the timeout to detect a frozen game
+    #
+    # Result::
+    # * Integer: Timeout to detect a frozen game
+    def timeout_frozen_tests_secs
+      @game_info['timeout_frozen_tests_secs']
+    end
+
+    # Return the timeout before restarting a game tests session
+    #
+    # Result::
+    # * Integer: Timeout before restarting a game tests session
+    def timeout_interrupt_tests_secs
+      @game_info['timeout_interrupt_tests_secs']
+    end
+
     # Return an xEdit instance for this game
     #
     # Result::
@@ -56,6 +92,88 @@ module Modsvaskr
       @xedit
     end
 
+    # Launch the game, and wait for launch to be successful
+    #
+    # Parameters::
+    # * *autoload* (Boolean or String): If false, then launch the game using the normal launcher. If String, then use AutoLoad to load a given saved file (or empty to continue latest save) [default: false].
+    def launch(autoload: false)
+      # Launch the game
+      @idx_launch = 0 unless defined?(@idx_launch)
+      if autoload
+        log "[ Game #{name} ] - Launch game (##{@idx_launch}) using AutoLoad #{autoload}..."
+        autoload_file = "#{path}/Data/AutoLoad.cmd"
+        if File.exist?(autoload_file)
+          run_cmd({
+            dir: path,
+            exe: 'Data\AutoLoad.cmd',
+            args: [autoload]
+          })
+        else
+          log "[ Game #{name} ] - Missing file #{autoload_file}. Can't use AutoLoad to load game automatically. Please install the AutoLoad mod."
+        end
+      else
+        log "[ Game #{name} ] - Launch game (##{@idx_launch}) using configured launcher (#{launch_exe})..."
+        run_cmd({
+          dir: path,
+          exe: launch_exe
+        })
+      end
+      @idx_launch += 1
+      # The game launches asynchronously, so just wait a little bit and check for the process existence
+      sleep @game_info['min_launch_time_secs']
+      tasklist_stdout = nil
+      loop do
+        tasklist_stdout = `tasklist | find "#{running_exe}"`.strip
+        break unless tasklist_stdout.empty?
+        log "[ Game #{name} ] - #{running_exe} is not running. Wait for its startup..."
+        sleep 1
+      end
+      @pid = Integer(tasklist_stdout.split(' ')[1])
+      log "[ Game #{name} ] - #{running_exe} has started with PID #{@pid}"
+    end
+
+    # Is the game currently running?
+    #
+    # Result::
+    # * Boolean: Is the game currently running?
+    def running?
+      if @pid
+        running = true
+        begin
+          # Process.kill does not work when the game has crashed (the process is still detected as zombie)
+          # running = Process.kill(0, @pid) == 1
+          tasklist_stdout = `tasklist | find "#{running_exe}"`.strip
+          running = !tasklist_stdout.empty?
+          # log "[ Game #{name} ] - Tasklist returned no #{running_exe}:\n#{tasklist_stdout}" unless running
+        rescue Errno::ESRCH
+          log "[ Game #{name} ] - Got error while waiting for #{running_exe} PID #{@pid}: #{$!}"
+          running = false
+        end
+        @pid = nil unless running
+        running
+      else
+        false
+      end
+    end
+
+    # Kill the game, and wait till it is killed
+    def kill
+      if @pid
+        first_time = true
+        while @pid do
+          system "taskkill #{first_time ? '' : '/F '}/pid #{@pid}"
+          first_time = false
+          sleep 1
+          if running?
+            log "[ Game #{name} ] - #{running_exe} is still running (PID #{@pid}). Wait for its kill..."
+            sleep 5
+          end
+        end
+      else
+        log "[ Game #{name} ] - Game not started, so nothing to kill."
+      end
+    end
+
   end
 
 end

data/lib/modsvaskr/games/skyrim_se.rb

@@ -9,11 +9,6 @@ module Modsvaskr
     # Handle a Skyrim installation
     class SkyrimSe < Game
 
-      SEVEN_ZIP_CMD = {
-        dir: 'C:\Program Files\7-Zip',
-        exe: '7z.exe'
-      }
-
       # Initialize the game
       # [API] - This method is optional
       def init
@@ -28,8 +23,8 @@ module Modsvaskr
       def complete_game_menu(menu)
         menu.item 'Install SKSE64' do
           install_skse64
-          puts 'Press Enter to continue...'
-          $stdin.gets
+          out 'Press Enter to continue...'
+          wait_for_user_enter
         end
       end
 
@@ -60,7 +55,7 @@ module Modsvaskr
 
       # Install SKSE64 corresponding to our game
       def install_skse64
-        doc = Nokogiri::HTML(open('https://skse.silverlock.org/'))
+        doc = Nokogiri::HTML(URI.open('https://skse.silverlock.org/'))
         p_element = doc.css('p').find { |el| el.text.strip =~ /^Current SE build .+: 7z archive$/ }
         if p_element.nil?
           log '!!! Can\'t get SKSE64 from https://skse.silverlock.org/. It looks like the page structure has changed. Please update the code or install it manually.'
@@ -69,14 +64,20 @@ module Modsvaskr
           path = "#{@tmp_dir}/skse64.7z"
           FileUtils.mkdir_p File.dirname(path)
           log "Download from #{url} => #{path}..."
-          open(url, 'rb') do |web_io|
+          URI.open(url, 'rb') do |web_io|
             File.write(path, web_io.read, mode: 'wb')
           end
           skse64_tmp_dir = "#{@tmp_dir}/skse64"
           log "Unzip into #{skse64_tmp_dir}..."
           FileUtils.rm_rf skse64_tmp_dir
           FileUtils.mkdir_p skse64_tmp_dir
-          run_cmd SEVEN_ZIP_CMD, args: ['x', "\"#{path}\"", "-o\"#{skse64_tmp_dir}\"", '-r']
+          run_cmd(
+            {
+              dir: @config.seven_zip_path,
+              exe: '7z.exe'
+            },
+            args: ['x', "\"#{path}\"", "-o\"#{skse64_tmp_dir}\"", '-r']
+          )
           skse64_subdir = Dir.glob("#{skse64_tmp_dir}/*").first
           log "Move files from #{skse64_subdir} to #{self.path}..."
           FileUtils.cp_r "#{skse64_subdir}/.", self.path, remove_destination: true

data/lib/modsvaskr/in_game_tests_runner.rb

@@ -0,0 +1,335 @@
+require 'base64'
+require 'elder_scrolls_plugin'
+require 'fileutils'
+require 'json'
+require 'time'
+require 'modsvaskr/tests_suite'
+
+module Modsvaskr
+
+  # Class getting a simple API to handle tests that are run in-game
+  class InGameTestsRunner
+
+    include Logger
+
+    # Constructor.
+    # Default values are for a standard Skyrim SE installation.
+    #
+    # Parameters::
+    # * *config* (Config): Main configuration
+    # * *game* (Game): Game for which we run tests
+    def initialize(config, game)
+      @config = config
+      @game = game
+      auto_test_esp = "#{@game.path}/Data/AutoTest.esp"
+      # Ordered list of available in-game test suites
+      # Array<Symbol>
+      @available_tests_suites =
+        if File.exist?(auto_test_esp)
+          Base64.decode64(
+            ElderScrollsPlugin.new(auto_test_esp).
+              to_json[:sub_chunks].
+              find { |chunk| chunk[:decoded_header][:label] == 'QUST' }[:sub_chunks].
+              find do |chunk|
+                chunk[:sub_chunks].any? { |sub_chunk| sub_chunk[:name] == 'EDID' && sub_chunk[:data] =~ /AutoTest_ScriptsQuest/ }
+              end[:sub_chunks].
+              find { |chunk| chunk[:name] == 'VMAD' }[:data]
+          ).scan(/AutoTest_Suite_(\w+)/).flatten.map { |tests_suite| tests_suite.downcase.to_sym }
+        else
+          log "[ In-game testing #{@game.name} ] - Missing file #{auto_test_esp}. In-game tests will be disabled. Please install the AutoTest mod."
+          []
+        end
+      log "[ In-game testing #{@game.name} ] - #{@available_tests_suites.size} available in-game tests suites: #{@available_tests_suites.join(', ')}"
+    end
+
+    # Run in-game tests in a loop until they are all tested
+    #
+    # Parameters::
+    # * *selected_tests* (Hash<Symbol, Array<String> >): Ordered list of in-game tests to be run, per in-game tests suite
+    # * Proc: Code called when a in-game test status has changed
+    #   * Parameters::
+    #     * *in_game_tests_suite* (Symbol): The in-game tests suite for which test statuses have changed
+    #     * *in_game_tests_statuses* (Hash<String,String>): Tests statuses, per test name
+    def run(selected_tests)
+      unknown_tests_suites = selected_tests.keys - @available_tests_suites
+      log "[ In-game testing #{@game.name} ] - !!! The following in-game tests suites are not supported: #{unknown_tests_suites.join(', ')}" unless unknown_tests_suites.empty?
+      tests_to_run = selected_tests.select { |tests_suite, _tests| !unknown_tests_suites.include?(tests_suite) }
+      unless tests_to_run.empty?
+        FileUtils.mkdir_p "#{@game.path}/Data/SKSE/Plugins/StorageUtilData"
+        tests_to_run.each do |tests_suite, tests|
+          # Write the JSON file that contains the list of tests to run
+          File.write(
+            "#{@game.path}/Data/SKSE/Plugins/StorageUtilData/AutoTest_#{tests_suite}_Run.json",
+            JSON.pretty_generate(
+              'stringList' => {
+                'tests_to_run' => tests
+              }
+            )
+          )
+          # Clear the AutoTest test statuses
+          File.unlink("#{@game.path}/Data/SKSE/Plugins/StorageUtilData/AutoTest_#{tests_suite}_Statuses.json") if File.exist?("#{@game.path}/Data/SKSE/Plugins/StorageUtilData/AutoTest_#{tests_suite}_Statuses.json")
+        end
+        auto_test_config_file = "#{@game.path}/Data/SKSE/Plugins/StorageUtilData/AutoTest_Config.json"
+        # Write the JSON file that contains the configuration of the AutoTest tests runner
+        File.write(
+          auto_test_config_file,
+          JSON.pretty_generate(
+            'string' => {
+              'on_start' => 'run',
+              'on_stop' => 'exit'
+            }
+          )
+        )
+        out ''
+        out '=========================================='
+        out '= In-game tests are about to be launched ='
+        out '=========================================='
+        out ''
+        out 'Here is what you need to do once the game will be launched (don\'t launch it by yourself, the test framework will launch it for you):'
+        out '* Load the game save you want to test (or start a new game).'
+        out ''
+        out 'This will execute all in-game tests automatically.'
+        out ''
+        out 'It is possible that the game crashes during tests:'
+        out '* That\'s a normal situation, as tests don\'t mimick a realistic gaming experience, and the Bethesda engine is not meant to be stressed like that.'
+        out '* In case of game crash (CTD), the Modsvaskr test framework will relaunch it automatically and resume testing from when it crashed.'
+        out '* In case of repeated CTD on the same test, the Modsvaskr test framework will detect it and skip the crashing test automatically.'
+        out '* In case of a game freeze without CTD, the Modsvaskr test framework will detect it after a few minutes and automatically kill the game before re-launching it to resume testing.'
+        out ''
+        out 'If you want to interrupt in-game testing: invoke the console with ~ key and type stop_tests followed by Enter.'
+        out ''
+        out 'Press enter to start in-game testing (this will lauch your game automatically)...'
+        wait_for_user_enter
+        last_time_tests_changed = nil
+        with_auto_test_monitoring(
+          on_auto_test_statuses_diffs: proc do |in_game_tests_suite, in_game_tests_statuses|
+            yield in_game_tests_suite, in_game_tests_statuses
+            last_time_tests_changed = Time.now
+          end
+        ) do
+          # Loop on (re-)launching the game when we still have tests to perform
+          idx_launch = 0
+          loop do
+            # Check which test is supposed to run first, as it will help in knowing if it fails or not.
+            first_tests_suite_to_run = nil
+            first_test_to_run = nil
+            current_tests_statuses = check_auto_test_statuses
+            @available_tests_suites.each do |tests_suite|
+              if tests_to_run.key?(tests_suite)
+                found_test_ok =
+                  if current_tests_statuses.key?(tests_suite)
+                    # Find the first test that would be run (meaning the first one having no status, or status 'started')
+                    tests_to_run[tests_suite].find do |test_name|
+                      found_test_name, found_test_status = current_tests_statuses[tests_suite].find { |(current_test_name, _current_test_status)| current_test_name == test_name }
+                      found_test_name.nil? || found_test_status == 'started'
+                    end
+                  else
+                    # For sure the first test of this suite will be the first one to run
+                    tests_to_run[tests_suite].first
+                  end
+                if found_test_ok
+                  first_tests_suite_to_run = tests_suite
+                  first_test_to_run = found_test_ok
+                  break
+                end
+              end
+            end
+            if first_tests_suite_to_run.nil?
+              log "[ In-game testing #{@game.name} ] - No more test to be run."
+              break
+            else
+              log "[ In-game testing #{@game.name} ] - First test to run should be #{first_tests_suite_to_run} / #{first_test_to_run}."
+              # Launch the game to execute AutoTest
+              @game.launch(autoload: idx_launch == 0 ? false : 'auto_test')
+              idx_launch += 1
+              log "[ In-game testing #{@game.name} ] - Start monitoring in-game testing..."
+              last_time_tests_changed = Time.now
+              while @game.running? do
+                check_auto_test_statuses
+                # If the tests haven't changed for too long, consider the game has frozen, but not crashed. So kill it.
+                if Time.now - last_time_tests_changed > @game.timeout_frozen_tests_secs
+                  log "[ In-game testing #{@game.name} ] - Last time in-game tests statuses have changed is #{last_time_tests_changed.strftime('%F %T')}. Consider the game is frozen, so kill it."
+                  @game.kill
+                else
+                  sleep @game.tests_poll_secs
+                end
+              end
+              last_test_statuses = check_auto_test_statuses
+              # Log latest statuses
+              log "[ In-game testing #{@game.name} ] - End monitoring in-game testing. In-game test statuses after game run:"
+              last_test_statuses.each do |tests_suite, statuses_for_type|
+                log "[ In-game testing #{@game.name} ] - [ #{tests_suite} ] - #{statuses_for_type.select { |(_name, status)| status == 'ok' }.size} / #{statuses_for_type.size}"
+              end
+              # Check for which reason the game has stopped, and eventually end the testing session.
+              # Careful as this JSON file can be written by Papyrus that treat strings as case insensitive.
+              # cf. https://github.com/xanderdunn/skaar/wiki/Common-Tasks
+              auto_test_config = Hash[JSON.parse(File.read(auto_test_config_file))['string'].map { |key, value| [key.downcase, value.downcase] }]
+              if auto_test_config.dig('stopped_by') == 'user'
+                log "[ In-game testing #{@game.name} ] - Tests have been stopped by user."
+                break
+              end
+              if auto_test_config.dig('tests_execution') == 'end'
+                log "[ In-game testing #{@game.name} ] - Tests have finished running."
+                break
+              end
+              # From here we know that the game has either crashed or has been killed.
+              # This is an abnormal termination of the game.
+              # We have to know if this is due to a specific test that fails deterministically, or if it is the engine being unstable.
+              # Check the status of the first test that should have been run to know about it.
+              first_test_status = nil
+              _found_test_name, first_test_status = last_test_statuses[first_tests_suite_to_run].find { |(current_test_name, _current_test_status)| current_test_name == first_test_to_run } if last_test_statuses.key?(first_tests_suite_to_run)
+              if first_test_status == 'ok'
+                # It's not necessarily deterministic.
+                # We just have to go on executing next tests.
+                log "[ In-game testing #{@game.name} ] - Tests session has finished in error, certainly due to the game's normal instability. Will resume testing."
+              else
+                # The first test doesn't pass.
+                # We need to mark it as failed, then remove it from the runs.
+                log "[ In-game testing #{@game.name} ] - First test #{first_tests_suite_to_run} / #{first_test_to_run} is in error status: #{first_test_status}. Consider it failed and skip it for next run."
+                # If the test was started but failed before setting its status to something else then change the test status in the JSON file directly so that AutoTest does not try to re-run it.
+                if first_test_status == 'started' || first_test_status == '' || first_test_status.nil?
+                  File.write(
+                    "#{@game.path}/Data/SKSE/Plugins/StorageUtilData/AutoTest_#{first_tests_suite_to_run}_Statuses.json",
+                    JSON.pretty_generate(
+                      'string' => Hash[((last_test_statuses[first_tests_suite_to_run] || []) + [[first_test_to_run, '']]).map do |(test_name, test_status)|
+                        [
+                          test_name,
+                          test_name == first_test_to_run ? 'failed_ctd' : test_status
+                        ]
+                      end]
+                    )
+                  )
+                  # Notify the callbacks updating test statuses
+                  check_auto_test_statuses
+                end
+              end
+              # We will start again. Leave some time to interrupt if we want.
+              out "We are going to start again in #{@game.timeout_interrupt_tests_secs} seconds. Press Enter now to interrupt it."
+              key_pressed =
+                begin
+                  Timeout.timeout(@game.timeout_interrupt_tests_secs) { $stdin.gets }
+                rescue
+                  nil
+                end
+              if key_pressed
+                log "[ In-game testing #{@game.name} ] - Run interrupted by user."
+                # TODO: Remove AutoTest start on load: it has been interrupted by the user, so we should not keep it in case the user launches the game by itself.
+                break
+              end
+            end
+          end
+        end
+      end
+    end
+
+    private
+
+    # Start an AutoTest monitoring session.
+    # This allows checking for test statuses differences easily.
+    #
+    # Parameters::
+    # * *on_auto_test_statuses_diffs* (Proc): Code called when a in-game test status has changed
+    #   * Parameters::
+    #     * *in_game_tests_suite* (Symbol): The in-game tests suite for which test statuses have changed
+    #     * *in_game_tests_statuses* (Hash<String,String>): Tests statuses, per test name
+    # * Proc: Code called with monitoring on
+    def with_auto_test_monitoring(on_auto_test_statuses_diffs:)
+      @last_auto_test_statuses = {}
+      @on_auto_test_statuses_diffs = on_auto_test_statuses_diffs
+      yield
+    end
+
+    # Check AutoTest test statuses differences, and call some code in case of changes in those statuses.
+    # Remember the last checked statuses to only find diffs on next call.
+    # Prerequisites: To be called inside a with_auto_test_monitoring block
+    #
+    # Result::
+    # * Hash<Symbol, Array<[String, String]> >: Ordered list of AutoTest [test name, test status], per AutoTest tests suite
+    def check_auto_test_statuses
+      # Log a diff in tests
+      new_statuses = auto_test_statuses
+      diff_statuses = diff_statuses(@last_auto_test_statuses, new_statuses)
+      unless diff_statuses.empty?
+        # Tests have progressed
+        log "[ In-game testing #{@game.name} ] - #{diff_statuses.size} tests suites have statuses changes:"
+        diff_statuses.each do |tests_suite, tests_statuses|
+          log "[ In-game testing #{@game.name} ]   * #{tests_suite}:"
+          tests_statuses.each do |(test_name, test_status)|
+            log "[ In-game testing #{@game.name} ]     * #{test_name}: #{test_status}"
+          end
+          @on_auto_test_statuses_diffs.call(tests_suite, Hash[tests_statuses])
+        end
+      end
+      # Remember the current statuses
+      @last_auto_test_statuses = new_statuses
+      @last_auto_test_statuses
+    end
+
+    # Get the list of AutoTest statuses
+    #
+    # Result::
+    # * Hash<Symbol, Array<[String, String]> >: Ordered list of AutoTest [test name, test status], per AutoTest tests suite
+    def auto_test_statuses
+      statuses = {}
+      `dir "#{@game.path}/Data/SKSE/Plugins/StorageUtilData" /B`.split("\n").each do |file|
+        if file =~ /^AutoTest_(.+)_Statuses\.json$/
+          auto_test_suite = $1.downcase.to_sym
+          # Careful as this JSON file can be written by Papyrus that treat strings as case insensitive.
+          # cf. https://github.com/xanderdunn/skaar/wiki/Common-Tasks
+          statuses[auto_test_suite] = JSON.parse(File.read("#{@game.path}/Data/SKSE/Plugins/StorageUtilData/#{file}"))['string'].map do |test_name, test_status|
+            [test_name.downcase, test_status.downcase]
+          end
+        end
+      end
+      statuses
+    end
+
+    # Diff between test statuses.
+    #
+    # Parameters::
+    # * *statuses1* (Hash<Symbol, Array<[String, String]> >): Test statuses, per test name (in a sorted list), per tests suite
+    # * *statuses2* (Hash<Symbol, Array<[String, String]> >): Test statuses, per test name (in a sorted list), per tests suite
+    # Result::
+    # * Hash<Symbol, Array<[String, String]> >: statuses2 - statuses1
+    def diff_statuses(statuses1, statuses2)
+      statuses = {}
+      statuses1.each do |tests_suite, tests_info|
+        if statuses2.key?(tests_suite)
+          # Handle Hashes as it will be faster
+          statuses1_for_test = Hash[tests_info]
+          statuses2_for_test = Hash[statuses2[tests_suite]]
+          statuses1_for_test.each do |test_name, status1|
+            if statuses2_for_test.key?(test_name)
+              if statuses2_for_test[test_name] != status1
+                # Change in status
+                statuses[tests_suite] = [] unless statuses.key?(tests_suite)
+                statuses[tests_suite] << [test_name, statuses2_for_test[test_name]]
+              end
+            else
+              # This test has been removed
+              statuses[tests_suite] = [] unless statuses.key?(tests_suite)
+              statuses[tests_suite] << [test_name, 'deleted']
+            end
+          end
+          statuses2_for_test.each do |test_name, status2|
+            unless statuses1_for_test.key?(test_name)
+              # This test has been added
+              statuses[tests_suite] = [] unless statuses.key?(tests_suite)
+              statuses[tests_suite] << [test_name, status2]
+            end
+          end
+        else
+          # All test statuses have been removed
+          statuses[tests_suite] = tests_info.map { |(test_name, _test_status)| [test_name, 'deleted'] }
+        end
+      end
+      statuses2.each do |tests_suite, tests_info|
+        # All test statuses have been added
+        statuses[tests_suite] = tests_info unless statuses1.key?(tests_suite)
+      end
+      statuses
+    end
+
+  end
+
+end