cloud-toaster 1.1.2

data/lib/toaster/test/test_generator.rb

@@ -0,0 +1,246 @@
+
+
+#
+# Author: Waldemar Hummer (hummer@dsg.tuwien.ac.at)
+#
+
+require "toaster/state/state_transition_graph"
+require "toaster/test/test_coverage"
+require "toaster/test/test_case"
+
+include Toaster
+
+module Toaster
+  class TestGenerator
+
+    attr_accessor :test_suite, :state_graph
+
+    def initialize(test_suite, state_graph = nil)
+      @state_graph = state_graph
+      @test_suite = test_suite
+      @automation = test_suite.automation
+      if !state_graph
+        @state_graph = StateTransitionGraph.build_graph_for_test_suite(@test_suite)
+      end
+      @coverage = TestCoverage.new(@test_suite, @state_graph)
+    end
+
+    def default_test_case()
+      c = TestCase.new(:test_suite => @test_suite)
+      return c
+    end
+
+    def gen_test_each_task(exclude_self=false)
+      result = []
+
+      tasks = @automation.tasks
+      # remove tasks that are related to the Chef instrumentation 
+      # performed by the Testing framework
+      tasks = tasks.select{ |t| !t.toaster_testing_task? }
+
+      tasks.each do |t|
+
+        parameter_combinations = gen_param_combinations(t.parameters)
+        puts "INFO: generated #{parameter_combinations.size} parameter combinations: #{parameter_combinations.inspect}"
+
+        parameter_combinations.each do |params|
+          c = TestCase.new(:test_suite => @test_suite)
+          skip_tasks = tasks.select{ |t1| exclude_self ? t1.uuid == t.uuid : t1.uuid != t.uuid }
+          skip_task_uuids = skip_tasks.collect{ |t1| t1.uuid }
+          c.skip_task_uuids.concat(skip_task_uuids)
+          c.test_attributes.merge(params)
+          if !result.include?(c) && !test_executed?(c)
+            result << c
+          end
+        end
+      end
+
+      return result
+    end
+
+    def gen_param_combinations(all_params, current={}, result=Set.new)
+      if all_params.empty?
+        result << current
+        return result
+      end
+      all_params_new = all_params.dup
+      this_param = all_params_new[0]
+      all_params_new.delete(this_param)
+      key = this_param.key
+
+      @test_suite.parameter_test_values[key].each do |val|
+        new_cur = current.clone()
+        new_cur[key] = val
+        gen_param_combinations(all_params_new, new_cur, result)
+      end
+      
+      return result
+    end
+
+    def gen_test_exclude_each_task()
+      gen_test_each_task(true)
+    end
+
+    def gen_all_tests()
+      cvg_goal = @test_suite.coverage_goal
+      if cvg_goal.graph == StateGraphCoverage::STATES
+        return gen_test_each_state()
+      elsif cvg_goal.graph == StateGraphCoverage::TRANSITIONS
+        return gen_test_each_transition()
+      end
+      puts "WARN: Coverage goal should be either STATES or TRANSITIONS."
+      return []
+    end
+
+    def gen_test_each_state(max_node_occurrences=2)
+      result = []
+      all_states = state_graph.nodes
+      covered_states = Set.new
+      covered_states << @state_graph.start_node
+      problem_states = []
+      while covered_states.size + problem_states.size < all_states.size
+        state_to_cover = (all_states - problem_states - covered_states).to_a[0]
+        path = do_create_path([state_to_cover], max_node_occurrences)
+        if !path
+          puts "INFO: problem state detected: #{state_to_cover}"
+          state_to_cover.outgoing.each do |e|
+            #puts "#{state_to_cover} --> #{e.node_to}"
+            e.node_to.outgoing.each do |e1|
+              #puts "#{e.node_to} ==> #{e1.node_to}"
+            end
+          end
+          state_to_cover.incoming.each do |e|
+            #puts "#{e.node_from} ~~> #{state_to_cover}"
+          end
+          problem_states << state_to_cover
+        else
+          covered_states.merge(path.collect { |e| e.node_to } )
+          tasks = get_task_sequence(path)
+          c = TestCase.new(:test_suite => @test_suite)
+          #puts "DEBUG: sequence of tasks for test case: #{tasks.collect {|t| t.uuid}}"
+          c.repeat_task_uuids = gen_repeat_config(tasks)
+          #puts "DEBUG: repeat tasks: #{c.repeat_task_uuids}"
+          params = {}
+          path.each do |edge|
+            # TODO! add parameters!
+            #params.merge!(edge.transition.parameters)
+          end
+          c.test_attributes.concat(RunAttribute.from_hash(params))
+          if !result.include?(c) && !test_executed?(c)
+            result << c
+          end
+        end
+        #puts "--> #{covered_states.size} of #{all_states.size} states covered by #{result.size} test cases"
+      end
+      return result
+    end
+
+    def gen_test_each_transition()
+      result = []
+      # TODO
+      return result
+    end
+
+    private
+
+    def do_create_path(nodes_to_include=[], max_node_occurrences=2,
+        path_so_far=[], target_node=@state_graph.end_node, visited=[])
+
+      current_node = path_so_far.empty? ? @state_graph.start_node : path_so_far[-1].node_to
+
+      visited << current_node
+
+      #puts "node #{current_node} - #{path_so_far[-1]} - #{path_so_far[-1].node_from}"
+      if current_node.kind_of?(StateNodeFinal) || current_node == target_node
+        #puts "nodes_to_include: #{nodes_to_include.size()}"
+        return nil if !nodes_to_include.empty? 
+        return path_so_far
+      end
+      puts "WARN: no outgoing edges for node #{current_node}!" if current_node.outgoing.empty?
+      current_node.outgoing.each do |edge|
+        if visited.count(edge.node_to) < max_node_occurrences
+          all_accessible = true
+          nodes_to_include.each do |node|
+            reachable = nil
+            begin 
+              reachable = edge.node_to.node_reachable?(node)
+            rescue Object => ex
+              puts caller.join("\n")
+              puts caller.size
+              puts visited.size
+              raise ex
+            end
+            #puts "INFO #{node} reachable from #{edge.node_to}: #{reachable} - #{current_node.outgoing.size} - #{visited.size}" if reachable
+            if !reachable
+              all_accessible = false
+              break
+            end
+          end
+          if all_accessible
+            path_so_far_copy = path_so_far.dup
+            path_so_far_copy << edge
+            nodes_to_include_copy = nodes_to_include.dup
+            del = nodes_to_include_copy.delete(edge.node_to)
+            #puts "INFO: deleted node #{edge.node_to}, pre-task #{@state_graph.tasks.index(edge.represented_task)} from list of nodes to include!" if del
+            path = do_create_path(nodes_to_include_copy, max_node_occurrences, path_so_far_copy, target_node, visited.dup)
+            return path if path
+          end
+        end
+      end
+      #puts "all_visited: #{all_visited.size}"
+      #puts "WARN: Looks like not all #{nodes_to_include.size} required nodes are accessible from node #{current_node}"
+      return nil
+      
+      # choose random successor node
+      index = rand(current_node.outgoing.size)
+      edge = current_node.outgoing.to_a[index]
+      path_so_far << edge
+      nodes_to_include = nodes_to_include.dup
+      nodes_to_include.delete(edge.node_to)
+      return do_create_path(nodes_to_include, max_node_occurrences, path_so_far, target_node, visited)
+    end
+
+    def get_task_sequence(transition_path)
+      seq = (transition_path.select { |e| !e.represented_task.nil? }).collect { |e| e.represented_task }
+      #puts "DEBUG: sequence of #{seq.size} tasks extracted from path with #{transition_path.size} transitions"
+      return seq
+    end
+
+    def test_executed?(test_case)
+      return @coverage.test_executed?(test_case)
+    end
+
+    def gen_repeat_config(task_list)
+      result = []
+      index = {}
+      i = 0
+      while i < task_list.size
+        task = task_list[i]
+        existing = index[task.uuid]
+        index[task.uuid] = i
+        if !existing.nil?
+          combination = task_list[existing..i-1].collect { |t| t ? t.uuid : nil }
+          result << combination
+          combination_length = combination.size
+          i += combination_length
+        else
+          i += 1
+        end
+      end
+      return result
+    end
+
+    def get_indexes(task, task_list)
+      index = 0
+      result = []
+      task_list.each do |t|
+        if task == t || task.eql?(t)
+          result << index
+        end
+        index += 1
+      end
+      return result
+    end
+
+  end
+end

data/lib/toaster/test/test_orchestrator.rb

@@ -0,0 +1,165 @@
+####################################################
+# (c) Waldemar Hummer (hummer@infosys.tuwien.ac.at)
+####################################################
+
+#
+# Author: Waldemar Hummer (hummer@dsg.tuwien.ac.at)
+#
+
+require "toaster/test/test_case"
+require "toaster/test/test_generator"
+require "toaster/api"
+
+include Toaster
+
+module Toaster
+  class TestOrchestrator
+
+    attr_accessor :uuid, :test_suite, :skip_task_uuids,
+    :repeat_task_uuids, :start_time, :end_time, :automation_run
+    def initialize(config = {})
+      @hosts = []
+      @host_proxies = {}
+      @round_robin_counter = -1
+    end
+
+    def add_host(host)
+      @hosts << host
+    end
+
+    def select_host()
+      raise "No hosts have been defined yet." if @hosts.empty?
+      @round_robin_counter = @round_robin_counter >= @hosts.size - 1 ? 0 : @round_robin_counter + 1
+      host = @hosts[@round_robin_counter]
+      client = get_proxy(host)
+      return client
+    end
+
+    def all_hosts()
+      result = []
+      @hosts.each do |host|
+        result << get_proxy(host)
+      end
+      return result
+    end
+
+    def update_all_services()
+      all_hosts.each do |h|
+        t = Thread.start {
+          h.update_code()
+        }
+      end
+      sleep 9
+    end
+
+    def exec_on_all_hosts(command, output=true, parallel=true)
+      out_total = ""
+      mutex = Mutex.new
+      block = lambda do |host|
+        p = get_proxy(host)
+        out_tmp = ""
+        out_tmp += "INFO: Executing command on #{host}: #{command}\n"
+        out = p.exec(command)
+        out_tmp += "#{out}\n"
+        mutex.synchronize do
+          out_total += "#{out_tmp}\n"
+        end
+        if output
+          puts out
+        end
+      end
+      if parallel
+        Util.exec_in_parallel(@hosts, &block)
+      else
+        @hosts.each &block
+      end
+      return out_total
+    end
+
+    def generate_tests(test_suite_uuid, coverage_goal)
+      suite = TestSuite.find({"uuid" => test_suite_uuid})[0]
+      suite.coverage_goal = coverage_goal
+      generate_tests_for_suite(suite)
+    end
+
+    def generate_tests_for_suite(test_suite)
+      gen = TestGenerator.new(test_suite)
+      puts "INFO: Starting to generate tests..."
+      tests = gen.gen_all_tests()
+      puts "INFO: Generated #{tests.size} test cases for test suite #{test_suite.uuid} (#{test_suite.test_cases.size} test cases so far)."
+      tests.each do |test|
+        if !test_suite.contains_equal_test?(test)
+          test_suite.test_cases << test
+        end
+      end
+      puts "INFO: Test suite #{test_suite.uuid} now contains #{test_suite.test_cases.size} test cases."
+      test_suite.save
+    end
+
+    def distribute_test_cases(tests_to_run)
+      tests = tests_to_run
+      tests_orig = tests_to_run.dup
+      puts "INFO: Distributing #{tests.size} generated tests to #{@hosts.size} hosts"
+      tests_by_host = {}
+      while !tests.empty?
+        test = tests.shift
+        host = select_host()
+        tests_by_host[host] = [] if !tests_by_host[host]
+        tests_by_host[host] << test.uuid
+        test.executing_host = host.host
+      end
+      tests_orig.each do |t|
+        # save test case to store "t.executing_host" to DB
+        t.save
+      end
+      tests_by_host.each do |host,test_case_list|
+        puts "INFO: Sending test case list #{test_case_list} to host #{host}"
+        blocking = false
+        output = host.runtest(test_case_list.join(","), blocking)
+      end
+    end
+
+    def distribute_tests(test_suite)
+      if !test_suite.kind_of?(TestSuite)
+        test_suite = TestSuite.find(:uuid => test_suite)[0]
+      end
+      tests = test_suite.test_cases
+      tests_to_run = []
+      tests.each do |t|
+        if !t.executed?()
+          tests_to_run << t
+        end
+      end
+      distribute_test_cases(tests_to_run)
+    end
+
+    def await_results(test_suite_uuid)
+      puts "INFO: Waiting for test results..."
+      suite = TestSuite.find({"uuid" => test_suite_uuid})[0]
+      unfinished_tests = suite.query_unfinished_tests
+      while !unfinished_tests.empty?
+        puts "INFO: Waiting some time for new test results (#{unfinished_tests.size} tests remaining)..."
+        sleep 10
+        unfinished_tests = suite.query_unfinished_tests
+      end
+    end
+
+    def clean_hosts()
+      Util.exec_in_parallel(all_hosts) do |h|
+        h.clean()
+      end
+    end
+
+    private
+
+    def get_proxy(host)
+      proxy = @host_proxies[host]
+      if !proxy
+        proxy = ToasterAppClient.new(host)
+        @host_proxies[host] = proxy
+      end
+      return proxy
+    end
+
+  end
+end

data/lib/toaster/test/test_result.rb

@@ -0,0 +1,73 @@
+
+#
+# Author: Waldemar Hummer (hummer@dsg.tuwien.ac.at)
+#
+
+require "toaster/util/timestamp"
+require "toaster/util/util"
+
+include Toaster
+
+module Toaster
+  class ResultEntry
+    attr_accessor :name, :value
+    def initialize(name=nil, value=nil)
+      @name = name
+      @value = value
+    end
+    def to_s
+      result = "<entry>\n<name>#{name}</name>\n<value>#{value}</value>\n</entry>\n"
+      return result
+    end
+  end
+
+  class IterationResult
+    attr_reader :entries
+    def initialize
+      @entries = []
+    end
+    def add(key, value)
+      add_entry(key, value)
+    end
+    def add_entry(key, value)
+      @entries << ResultEntry.new(key, value)
+    end
+    def to_s
+      result = "<iterations>\n"
+      entries().each do |e|
+        result += e.to_s
+      end
+      result += "</iterations>\n"
+      return result
+    end
+  end
+
+  class TestResult
+    attr_reader :iterations
+    attr_accessor :start_time, :end_time
+    def initialize
+      @iterations = []
+      start_time = TimeStamp.now.to_i
+    end
+
+    def new_iteration()
+      i = IterationResult.new()
+      @iterations << i
+      return i
+    end
+    def save(file_path)
+      content = to_s()
+      Util.write(file_path, content, true)
+    end
+
+    def to_s
+      result = "<genericTestResult>\n"
+      @iterations.each do |i|
+        result += i.to_s
+      end
+      result += "<startTime>#{start_time}</startTime>\n<finishTime>#{end_time}</finishTime>\n"
+      result += "</genericTestResult>"
+      return result
+    end
+  end
+end