testability-driver-runner 1.0.4 → 1.1.0

data/lib/tdrunner.rb

@@ -23,6 +23,7 @@ require File.expand_path( File.join( File.dirname( __FILE__ ), 'tdrunner_profile
 require File.expand_path( File.join( File.dirname( __FILE__ ), 'tdrunner_command_tool' ) )
 require 'yaml'
 
+
 #Sophisticated Interface for Running Random Automation
 module TDRunner
   #Main class which is used to start TDRunner run
@@ -43,16 +44,55 @@ module TDRunner
     #-p loads a execution profile from siearra.yml configuration file form config folder in the execution directory
     #parse the command line arguments if above steps are not found
     def initialize(args)
-      if args[0]=='-h'
+
+      if args.empty?
         tdrunner_help()
-      elsif args[0]=='--help'
+        return
+      end
+
+      verbose = 0
+
+      while true
+
+        if args[0]=='--display'
+          args.shift
+          ENV['DISPLAY'] = args.shift
+
+        elsif args[0]=='-h' or args[0]=='--help'
+          args.shift
+          tdrunner_help()
+
+        elsif args[0]=='--cd'
+          args.shift
+          Dir.chdir(args.shift)
+
+        elsif args[0]=='--verbose'
+          args.shift
+          verbose += 1
+
+        elsif args[0]=='--version'
+          args.shift
+          puts '1.1.0'
+
+        else
+          break
+        end
+      end
+
+      if verbose > 0
+        puts "DISPLAY=#{ENV['DISPLAY']}"
+        puts "PWD=#{Dir.pwd()}"
+      end
+
+      if args.empty?
         tdrunner_help()
-      elsif args[0]=='--version'
-        puts '1.0.0'
+
       elsif args[0]=='--websi'
         start_websi(args[1, args.size])
+
       elsif args[0]=='--cmd'
         tool = TDRunnerCommandTool.new(args[1, args.size])
+
       elsif args[0]=='-p'
         profile = TDRunnerProfileLoader.new(args)
         tdrunner_profiles=profile.get_execution_profiles
@@ -60,11 +100,11 @@ module TDRunner
           tdrunner_profile_execution_args=profile.args_from(tdrunner_profile)
           start_tdrunner(tdrunner_profile_execution_args)
         end
-      elsif args[0]==nil
-        tdrunner_help()
+
       else
         start_tdrunner(args)
       end
+
     end
 
     #Start tdrunner web UI
@@ -104,14 +144,16 @@ module TDRunner
       end
       @tdrunner_runner=TDRunnerTestUnit::ForkedTestUnitRunner.new(test_suite)
     end
+
     def execute_cucumber_tests(test_parameters=[])
       if Gem.available?('cucumber')
         @cucumber_command=Array.new(test_parameters)
         @tdrunner_runner=TDRunnerCucumber::ForkedCucumberRunner.new(@cucumber_command).execute
       else
-        $stderr.puts('WARNING: Cucumber test framework gem was not found or installed')
+        puts('WARNING: Cucumber test framework gem was not found or installed')
       end
     end
+
     def start_tdrunner(args)
       $tdrunner_parameters=TDRunnerParameters.new(args)
       ARGV.clear
@@ -133,15 +175,15 @@ module TDRunner
         @test_folder.each do |folder|
           if File.directory?(folder)
             if Gem.available?('cucumber')
-              if Gem.available?('cucumber','>0.4.4')
-                cucumber_parameters=["#{folder.to_s}", "-f", "TDriverReport::CucumberReporter"]
-              end
               if Gem.available?('cucumber','=0.4.4')
                 cucumber_parameters=["#{folder.to_s}", "-f", "TDriverReport::CucumberListener"]
               end
+              if Gem.available?('cucumber','>0.4.4')
+                cucumber_parameters=["#{folder.to_s}", "-f", "TDriverReport::CucumberReporter"]
+              end
               execute_cucumber_tests(cucumber_parameters)
             else
-              $stderr.puts('WARNING: Cucumber test framework gem was not found or installed')
+              puts('WARNING: Cucumber test framework gem was not found or installed')
             end
           end
         end
@@ -149,23 +191,25 @@ module TDRunner
 
       write_failed_sip_file($tdrunner_parameters)
     end
+
     #Method for displaying tdrunner help in commandline
     def tdrunner_help()
-      puts 'Usage: tdrunner [options] [framework command]'
+      puts 'Usage: tdrunner [runner opts] [--websi opts|--cmd opts|test opts] [command]'
       puts ''
       puts 'Examples:'
       puts ''
+      puts 'Execute tests in my_test_folder:'
       puts 'tdrunner my_test_folder'
       puts ''
-      puts 'Executes tests for 1.5 hours:'
+      puts 'Execute tests for 1.5 hours:'
       puts 'tdrunner -H 1.5 cucumber features'
       puts 'tdrunner -H 1.5 test_suite ts_my_test_unit_suite.rb'
       puts ''
-      puts 'Repeats tests for ten iterations:'
+      puts 'Repeat tests for ten iterations:'
       puts 'tdrunner -i 10 cucumber features'
       puts 'tdrunner -i 10 test_suite ts_my_test_unit_suite.rb'
       puts ''
-      puts 'Runs tests until defined date and time is reached:'
+      puts 'Run tests until defined date and time is reached:'
       puts 'tdrunner -d "12.12.2100 12:00" cucumber features'
       puts 'tdrunner -d "12.12.2100 12:00" test_suite ts_my_test_unit_suite.rb'
       puts ''
@@ -198,6 +242,38 @@ module TDRunner
       puts ''
       puts ''
 
+      puts "Runner options:"
+      puts ''
+
+      puts "--cd directory\tSet working directory, useful when invoking tdrunner remotely"
+
+      puts "--display display\tSet DISPLAY environment variable, useful for testing"
+      puts "\t\tdesktop applications remotely"
+
+      puts "-h, --help\tYou're looking at it."
+
+      puts "--verbose\tPrint extra information"
+
+      puts "--version\tShow version."
+
+      puts ''
+      puts ''
+      puts "--cmd sut sut_id httpd server_port [usb powerup|powerdown|reset] [status]"
+      puts "\t\tProbe status of given sut and given httpd server."
+      puts "\t\t(Used by tdrunner_ui.)"
+      puts "--cmd sut sut_id usb powerup|powerdown|reset"
+      puts "\t\tPerform given usb command using given sut."
+      puts "\t\t(Used by tdrunner_ui.)"
+      puts ''
+      puts ''
+      puts "--websi [optional arguments]"
+      puts "\t\tStarts tdrunner web interface, by default at URL"
+      puts "\t\thttp://localhost:3000/websi/"
+      puts ''
+      puts ''
+      puts "Test options:"
+      puts ''
+
       puts "-H i\t\tDefines how many hours tdrunner will run i as number of hours."
 
       puts "-i i\t\tDefines how many iterations tdrunner will run the tests i as"
@@ -214,27 +290,15 @@ module TDRunner
       puts "-t execution_profile"
       puts "\t\tCreate execution profile of failed Test::Unit tests"
 
-      puts "--cmd sut sut_id httpd server_port [usb powerup|powerdown|reset] [status]"
-      puts "\t\tProbe status of given sut and given httpd server."
-      puts "\t\t(Used by tdrunner_ui.)"
-      puts "--cmd sut sut_id usb powerup|powerdown|reset"
-      puts "\t\tPerform given usb command using given sut."
-      puts "\t\t(Used by tdrunner_ui.)"
-
       puts "--teardown\tRun test unit teardown only when the testcase is failed."
 
       puts "--ordered\tUse with execution profile to run tests as they are ordered"
       puts "\t\tin the sip file."
 
-      puts "--websi [optional arguments]"
-      puts "\t\tStarts tdrunner web interface, by default at URL"
-      puts "\t\thttp://localhost:3000/websi/"
-
       puts "-p profile_name\tPull commandline arguments from tdrunner.yml which can be"
       puts "\t\tdefined as strings or arrays."
 
-      puts "--version\tShow version."
-      puts "-h, --help\tYou're looking at it."
+
     end
   end
 end

data/lib/tdrunner_monitor.rb

@@ -22,60 +22,41 @@ module TDRunner
   #TDRunner monitor module for controlling the SIERRA execution
   module TDRunnerMonitor
 
-    def add_failed_testunit_tests(tdrunner_parameters,case_contents_array,index)
+    def add_failed_testunit_tests(tdrunner_parameters,case_contents_array)
       if(tdrunner_parameters.failed_execution_profile != nil)
-        failed_case = case_contents_array[index].at(0).to_s
-        if tdrunner_parameters.tdrunner_ordered==false
-          #i.e. no dependencies
-          add_tc_to_failed_array(tdrunner_parameters,failed_case)
-        else
-          add_dependent_tcs_to_failed_array(tdrunner_parameters,case_contents_array,index)
-        end
-      end
-    end
 
-    def add_dependent_tcs_to_failed_array(tdrunner_parameters,case_contents_array,index)
-      temp_array=Array.new
+        add_tcs_to_failed_array(case_contents_array)
 
-      failed_case = case_contents_array[index].at(0).to_s
-      pos = case_contents_array[index].at(2).count( "-").to_i
-      temp_array<<get_pos_string(pos) + get_test_name(failed_case)+ "(" + get_test_class(failed_case) + ")=1"
-      i = index.to_i - 1
+      end
+    end
 
-      while( i>0 ||  i==0 )
-        prev_pos = case_contents_array[i].at(2).count( "-").to_i
-        prev_failed_case = case_contents_array[i].at(0).to_s
+    def add_tcs_to_failed_array(case_contents_array)
 
-        if (pos>prev_pos && prev_pos!=0 )
-          temp_array<<get_pos_string(prev_pos) + get_test_name(prev_failed_case)+ "(" + get_test_class(prev_failed_case) + ")=1"
+      failed_cases=[]
+      $failed_tests ||= Array.new
+      b_fail=false
+      failed_level=0
+      class_name = get_test_class(case_contents_array[0].at(0).to_s) + "=1" if case_contents_array.count > 0
+      $failed_tests<<class_name if case_contents_array[0].at(0).to_s.include? '('
+
+      case_contents_array.reverse_each do |tcase|
+        if tcase.at(3)==false || tcase.at(3)=="not run"
+          class_name = get_test_class( tcase.at(2)) + "=1"
+          failed_level=tcase.at(2).count( "-").to_i
+          failed_cases << tcase.at(2)+ "=1" if tcase.at(2).include? '('
+          b_fail=true if failed_level!=0
         end
-
-        if(prev_pos==0)
-          temp_array<<get_pos_string(prev_pos) + get_test_name(prev_failed_case)+ "(" + get_test_class(prev_failed_case) + ")=1"
-          break
+        if b_fail==true and tcase.at(2).count( "-").to_i < failed_level and tcase.at(2).count( "-").to_i != 0 then
+          failed_cases << tcase.at(2)
+        elsif tcase.at(2).count( "-").to_i == 0 && b_fail==true
+          class_name = get_test_class(tcase.at(2)) + "=1"
+          failed_cases << tcase.at(2)+ "=1" if tcase.at(2).include? '('
+          b_fail=false
         end
-        i=i-1
       end
 
-      $failed_tests ||= Array.new
-      if !$failed_tests.include?  temp_array.reverse!
-        class_name = get_test_class(failed_case) + "=1"
-        $failed_tests <<class_name if !value_exists_in_array($failed_tests , class_name)
-        $failed_tests<<temp_array
-      end
-    end
+      $failed_tests<<failed_cases.reverse
 
-    def add_tc_to_failed_array(tdrunner_parameters,testcase)
-      $failed_tests ||= Array.new
-      tc_class=get_test_class(testcase)
-      tc_name=get_test_name(testcase)
-      class_name = tc_class + "=1"
-      tc_name = tc_name+ "(" + tc_class + ")=1"
-
-      if !$failed_tests.include? tc_name
-        $failed_tests <<class_name if !$failed_tests.include? class_name
-        $failed_tests << tc_name
-      end
     end
 
     def get_pos_string(pos)
@@ -134,10 +115,8 @@ module TDRunner
       secs=0
       if (/^\d+:\d+$/.match(duration_value.to_s))
         secs = (duration_value.split(':') [0].to_i*3600) + (duration_value.split(':') [1].to_i*60)
-      elsif (/^\d+\.\d+$/.match(duration_value.to_s))
+      elsif (/^\d+\.\d+$/.match(duration_value.to_s) || /^\d+$/.match(duration_value.to_s) )
         secs = 3600*duration_value.to_f
-      else (/^\d+$/.match(duration_value.to_s))
-        secs=60*duration_value.to_f
       end
       secs
     end
@@ -459,8 +438,10 @@ module TDRunner
 
       if tdrunner_parameters.tdrunner_ordered==false
         if b_profile_file_exist==true
-          @case_contents_array=@case_contents_array.shuffle
+          @case_contents_array=@case_contents_array.sort_by { rand }
           tests,@case_contents_array=order_the_features(tests, @case_contents_array)
+        else
+          tests=tests.sort_by { rand }
         end
       else
         if b_profile_file_exist==true
@@ -501,8 +482,10 @@ module TDRunner
 
       if tdrunner_parameters.tdrunner_ordered==false
         if b_profile_file_exist==true
-          @case_contents_array=@case_contents_array.shuffle
+          @case_contents_array=@case_contents_array.sort_by { rand }
           tests,@case_contents_array=order_the_tests(tests, @case_contents_array)
+        else
+          tests=tests.sort_by { rand }
         end
       else
         if b_profile_file_exist==true

data/lib/tdrunner_test_unit.rb

@@ -275,22 +275,48 @@ begin
           yield(STARTED, name)
           @_result = result
           @case_contents_array=case_contents_array
+          test_skipped = false
           begin
 
-            if self.respond_to?("setup_#{name[0..(name.to_s.index('(')-1)]}")
-              __send__("setup_#{name[0..(name.to_s.index('(')-1)]}") if status!=false
-            else
-              setup if status!=false
+            if MobyUtil::Parameter[ :runner_skip_enabled, "false" ] == "true" 
+						
+              fail_execs = tdriver_get_current_by_status( "failed" )
+              pass_execs = tdriver_get_current_by_status( "passed" )
+      
+              fail_execs = fail_execs.respond_to?( :count ) ? fail_execs.count : 0            
+              pass_execs = pass_execs.respond_to?( :count ) ? pass_execs.count : 0
+              
+              skip_treshold = MobyUtil::Parameter[ :runner_skip_treshold, "5" ].to_i
+              
+              if fail_execs + pass_execs >= skip_treshold and fail_execs + pass_execs > 0
+                skip_rate = MobyUtil::Parameter[ :runner_skip_fail_rate, "66" ].to_i / 100.0
+                test_skipped = fail_execs/(fail_execs + pass_execs).to_f >= skip_rate               
+              end
+            
             end
+            
+            if !test_skipped
+
+              if self.respond_to?("setup_#{name[0..(name.to_s.index('(')-1)]}")
+                __send__("setup_#{name[0..(name.to_s.index('(')-1)]}") if status!=false
+              else
+                setup if status!=false
+              end
+
+              if arguments!=nil && status==nil
+                __send__(@method_name,*arguments)
+              elsif status==false
+                Kernel::raise('Previous dependency test failed')
+              else
+                __send__(@method_name)
+              end
 
-            if arguments!=nil && status==nil
-              __send__(@method_name,*arguments)
-            elsif status==false
-              Kernel::raise('Previous dependency test failed')
             else
-              __send__(@method_name)
-            end
 
+              tdriver_report_log( "Not run due to consistent failures." )              
+              tdriver_report_set_test_case_status( MobyUtil::Parameter[ :runner_skip_status, "not run" ] )              
+              
+            end
 
           rescue AssertionFailedError => e
             add_failure(e.message, e.backtrace)
@@ -298,28 +324,30 @@ begin
             raise if PASSTHROUGH_EXCEPTIONS.include? $!.class
             add_error($!)
           ensure
-            begin
-              if $tdrunner_parameters.tdrunner_teardown==true
-                if @test_passed != true                  
+            unless test_skipped
+              begin
+                if $tdrunner_parameters.tdrunner_teardown==true
+                  if @test_passed != true                  
+                    if respond_to?("teardown_#{name[0..(name.to_s.index('(')-1)]}")
+                      __send__("teardown_#{name[0..(name.to_s.index('(')-1)]}") if status!=false
+                    else
+                      teardown if status!=false
+                    end
+                  end
+                else
                   if respond_to?("teardown_#{name[0..(name.to_s.index('(')-1)]}")
                     __send__("teardown_#{name[0..(name.to_s.index('(')-1)]}") if status!=false
                   else
                     teardown if status!=false
                   end
                 end
-              else
-                if respond_to?("teardown_#{name[0..(name.to_s.index('(')-1)]}")
-                  __send__("teardown_#{name[0..(name.to_s.index('(')-1)]}") if status!=false
-                else
-                  teardown if status!=false
-                end
+              rescue AssertionFailedError => e
+                add_failure(e.message, e.backtrace)
+              rescue Exception
+                raise if PASSTHROUGH_EXCEPTIONS.include? $!.class
+                add_error($!)
               end
-            rescue AssertionFailedError => e
-              add_failure(e.message, e.backtrace)
-            rescue Exception
-              raise if PASSTHROUGH_EXCEPTIONS.include? $!.class
-              add_error($!)
-            end
+			end
             @case_contents_array[current_index]=[@case_contents_array[current_index].at(0),@case_contents_array[current_index].at(1),@case_contents_array[current_index].at(2),@test_passed] if @case_contents_array!=nil && status==nil
           end
           result.add_run
@@ -418,12 +446,10 @@ begin
             else
               test.run(result, eval_parameters_arr, @case_contents_array, current_index, &progress_block)
             end
- 
-            if (@case_contents_array!=nil && @case_contents_array[current_index]!=nil  && @case_contents_array[current_index].at(3)==false)
-              add_failed_testunit_tests($tdrunner_parameters, @case_contents_array, current_index)
-            end
+             
             current_index+=1
           end
+          add_failed_testunit_tests($tdrunner_parameters, @case_contents_array)
           yield(FINISHED, name)
         end
       end

data/rakefile

@@ -1,7 +1,7 @@
 #
 # rakefile for building and releasing tdrunner
 
-@__release_mode = ENV['rel_mode']
+@__release_mode = 'release'
 @__release_mode = 'minor' if @__release_mode == nil
 GEM_NAME='testability-driver-runner'
 
@@ -105,6 +105,7 @@ task :gem_uninstall do
   puts "### Uninstalling GEM #{GEM_NAME}     ###"
   puts "#########################################################"
   tdrunner_gem = "tdrunner-#{@__gem_version}.gem"
+   FileUtils.rm(Dir.glob('pkg/*gem'))
   if /win/ =~ RUBY_PLATFORM    
     cmd = "del /F /Q pkg\\"
     failure = system(cmd)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: testability-driver-runner
 version: !ruby/object:Gem::Version 
-  version: 1.0.4
+  version: 1.1.0
 platform: ruby
 authors: 
 - Testability Driver team
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-02-08 00:00:00 +02:00
+date: 2011-03-11 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -64,78 +64,78 @@ extra_rdoc_files:
 files: 
 - readme
 - rakefile
-- bin/tdrunner
 - bin/sierra
-- lib/tdrunner_file_finder.rb
-- lib/tdrunner_profile.rb
-- lib/tdrunner_cucumber_runners.rb
+- bin/tdrunner
 - lib/tdrunner.rb
-- lib/tdrunner_command_tool.rb
 - lib/tdrunner.yml
-- lib/tdrunner_monitor.rb
+- lib/tdrunner_command_tool.rb
 - lib/tdrunner_cucumber.rb
+- lib/tdrunner_cucumber_runners.rb
+- lib/tdrunner_file_finder.rb
+- lib/tdrunner_monitor.rb
+- lib/tdrunner_profile.rb
 - lib/tdrunner_test_unit.rb
-- websi/script/about
-- websi/script/performance/benchmarker
-- websi/script/performance/profiler
-- websi/script/console
-- websi/script/destroy
-- websi/script/generate
-- websi/script/runner
-- websi/script/plugin
-- websi/script/server
-- websi/script/dbconsole
-- websi/log/server.log
-- websi/log/development.log
-- websi/log/production.log
-- websi/log/test.log
-- websi/config/environments/development.rb
-- websi/config/environments/production.rb
-- websi/config/environments/test.rb
-- websi/config/routes.rb
+- websi/app/controllers/application_controller.rb
+- websi/app/controllers/report_editor/test_run/cases_controller.rb
+- websi/app/controllers/report_editor_controller.rb
+- websi/app/controllers/websi_controller.rb
+- websi/app/controllers/websi_script.rb
+- websi/app/controllers/websi_support.rb
+- websi/app/helpers/application_helper.rb
+- websi/app/helpers/report_editor/report_editor_helper.rb
+- websi/app/helpers/report_editor/test_run/cases_helper.rb
+- websi/app/helpers/websi_helper.rb
+- websi/app/views/layouts/application.rhtml
+- websi/app/views/websi/execution.html.erb
+- websi/app/views/websi/index.html.erb
+- websi/app/views/websi/profile.html.erb
+- websi/app/views/websi/results.html.erb
+- websi/app/views/websi/tests.html.erb
+- websi/app/views/websi/weights.html.erb
 - websi/config/boot.rb
 - websi/config/database.yml
 - websi/config/environment.rb
-- websi/config/locales/en.yml
+- websi/config/environments/development.rb
+- websi/config/environments/production.rb
+- websi/config/environments/test.rb
+- websi/config/initializers/backtrace_silencers.rb
 - websi/config/initializers/inflections.rb
 - websi/config/initializers/mime_types.rb
 - websi/config/initializers/new_rails_defaults.rb
-- websi/config/initializers/backtrace_silencers.rb
 - websi/config/initializers/session_store.rb
+- websi/config/locales/en.yml
+- websi/config/routes.rb
 - websi/db/development.sqlite3
 - websi/db/seeds.rb
-- websi/README
-- websi/public/stylesheets/tdriver_report_style.css
+- websi/doc/README_FOR_APP
+- websi/log/development.log
+- websi/log/production.log
+- websi/log/server.log
+- websi/log/test.log
 - websi/public/report_editor/test_run/_index.html
 - websi/public/robots.txt
-- websi/public/tests/example_profile.sip
+- websi/public/stylesheets/tdriver_report_style.css
 - websi/public/tests/config/web_profile.sip
-- websi/public/tests/websi_parameters.xml
+- websi/public/tests/example_profile.sip
 - websi/public/tests/tdrunner.yml
+- websi/public/tests/websi_parameters.xml
 - websi/public/tests/web_profile.sip
-- websi/app/views/websi/results.html.erb
-- websi/app/views/websi/profile.html.erb
-- websi/app/views/websi/tests.html.erb
-- websi/app/views/websi/weights.html.erb
-- websi/app/views/websi/execution.html.erb
-- websi/app/views/websi/index.html.erb
-- websi/app/views/layouts/application.rhtml
-- websi/app/controllers/websi_controller.rb
-- websi/app/controllers/application_controller.rb
-- websi/app/controllers/report_editor/test_run/cases_controller.rb
-- websi/app/controllers/report_editor_controller.rb
-- websi/app/controllers/websi_support.rb
-- websi/app/controllers/websi_script.rb
-- websi/app/helpers/websi_helper.rb
-- websi/app/helpers/report_editor/test_run/cases_helper.rb
-- websi/app/helpers/report_editor/report_editor_helper.rb
-- websi/app/helpers/application_helper.rb
+- websi/Rakefile
+- websi/README
+- websi/script/about
+- websi/script/console
+- websi/script/dbconsole
+- websi/script/destroy
+- websi/script/generate
+- websi/script/performance/benchmarker
+- websi/script/performance/profiler
+- websi/script/plugin
+- websi/script/runner
+- websi/script/server
+- websi/test/functional/websi_controller_test.rb
 - websi/test/performance/browsing_test.rb
 - websi/test/test_helper.rb
-- websi/test/functional/websi_controller_test.rb
 - websi/test/unit/helpers/websi_helper_test.rb
-- websi/Rakefile
-- websi/doc/README_FOR_APP
 has_rdoc: true
 homepage: 
 licenses: []