rutema 2.0.0 → 2.0.1

data/lib/rutema/core/reporter.rb

@@ -1,19 +1,42 @@
-#  Copyright (c) 2007-2017 Vassilis Rizopoulos. All rights reserved.
+#  Copyright (c) 2007-2021 Vassilis Rizopoulos. All rights reserved.
+
 module Rutema
-  #Rutema supports two kinds of reporters.
+  ##
+  # Module for the definition of reporter classes
   #
-  #Block (from en bloc) reporters receive data via the report() method at the end of a Rutema run
-  #while event reporters receive events continuously during a run via the update() method
+  # rutema supports two kinds of reporters. Reporters derived from BlockReporter
+  # are supposed to receive data via the #report method at the end of a Rutema
+  # run. Reporters derived from EventReporter are intended to receive events
+  # continuously during a run via the #update method.
   #
-  #Nothing prevents you from creating a class that implements both behaviours
+  # Nothing permits implementing a reporter class which supports both
+  # behaviours.
   module Reporters
+    ##
+    # Base class for block reporters
+    #
+    # Block reporters are invoked at the end/after a test run. They offer means
+    # to e.g. print a summary after a test run or create a test report file for
+    # CI integration (see Reporters::JUnit as an example).
     class BlockReporter
+      ##
+      # Initialize a new instance from the given configuration
+      #
+      # * +configuration+ - the Configuration instance of the test run
+      # * +dispatcher+ - unused
       def initialize configuration,dispatcher
         @configuration=configuration
       end
+
+      ##
+      #
       def report specifications,states,errors
       end
     end
+
+    ##
+    # Event reporters receive and process information continually during a test
+    # run
     class EventReporter
       def initialize configuration,dispatcher
         @configuration=configuration
@@ -23,16 +46,13 @@ module Rutema
       def run!
         @thread=Thread.new do
           while true do
-            if  @queue.size>0
-              data=@queue.pop
-              begin
-                update(data) if data
-              rescue
-                puts "#{self.class} failed with #{$!.message}"
-                raise
-              end
+            data=@queue.pop
+            begin
+              update(data) if data
+            rescue
+              puts "#{self.class} failed with #{$!.message}"
+              raise
             end
-            sleep 0.1
           end
         end
       end
@@ -94,8 +114,10 @@ module Rutema
           when RunnerMessage
             if message.status == :error
               puts "FATAL|#{message.to_s}"
+            elsif message.status == :warning
+              puts "WARNING|#{message.to_s}"
             else
-              puts message.to_s if @mode=="verbose"
+              puts "#{message.to_s} #{message.status}." if @mode=="verbose"
             end
           when ErrorMessage
             puts message.to_s 
@@ -116,7 +138,8 @@ module Rutema
         states.each{|k,v| failures<<v.test if v.status==:error}
 
         unless @silent
-          puts "#{errors.size} errors. #{states.size} test cases executed. #{failures.size} failed"
+          count_tests_run = states.select { |name, state| !state.is_special }.count
+          puts "#{errors.size} errors. #{count_tests_run} test cases executed. #{failures.size} failed"
           unless failures.empty?
             puts "Failures:"
             puts specs.map{|spec| "  #{spec.name} - #{spec.filename}" if failures.include?(spec.name)}.compact.join("\n")

data/lib/rutema/core/runner.rb

@@ -1,4 +1,4 @@
-#  Copyright (c) 2007-2017 Vassilis Rizopoulos. All rights reserved.
+#  Copyright (c) 2007-2021 Vassilis Rizopoulos. All rights reserved.
 
 require_relative "framework"
 
@@ -6,6 +6,7 @@ module Rutema
   module Runners
     class Default
       include Rutema::Messaging
+
       attr_reader :context
       attr_accessor :setup,:teardown
       def initialize context,queue
@@ -14,41 +15,67 @@ module Rutema
         @context=context || Hash.new
         @queue = queue
         @number_of_runs=0
+        @cleanup_blocks = []
       end
 
-      def run spec
-        steps=[]
-        status=:success
-        state={'start_time'=>Time.now, "sequence_id"=>@number_of_runs,:test=>spec.name}
-        message(:test=>spec.name,:text=>'started')
-        if @setup
-          message(:test=>spec.name,:text=>'setup')
-          executed_steps,status=run_scenario("_setup_",@setup.scenario,@context)
-          steps+=executed_steps
-        end
-        if status!=:error
-          message(:test=>spec.name,:text=>'running')
-          executed_steps,status=run_scenario(spec.name,spec.scenario,@context)
-          steps+=executed_steps
-        else
-          message(:test=>spec.name,'number'=>0,'status'=>:error,'out'=>"Setup failed",'err'=>"",'duration'=>0)
-        end
-        state['status']=status
-        if @teardown
-          message(:test=>spec.name,:text=>'teardown')
-          executed_steps,status=run_scenario("_teardown_",@teardown.scenario,@context)
+      def run(spec, is_special = false)
+        begin
+          @context["spec_name"]=spec.name
+          steps=[]
+          status=:success
+          state={'start_time'=>Time.now, "sequence_id"=>@number_of_runs,:test=>spec.name}
+          message(:test=>spec.name,:text=>'started')
+          if @setup
+            message(:test=>spec.name,:text=>'setup')
+            executed_steps,setup_status = run_scenario("_setup_", @setup.scenario, @context, true)
+            status=setup_status unless STATUS_CODES.find_index(setup_status) < STATUS_CODES.find_index(status)
+            steps+=executed_steps
+          end
+          if status!=:error
+            message(:test=>spec.name,:text=>'running')
+            executed_steps,testspec_status = run_scenario(spec.name, spec.scenario, @context, is_special)
+            status=testspec_status unless STATUS_CODES.find_index(testspec_status) < STATUS_CODES.find_index(status)
+            steps+=executed_steps
+          else
+            message(:test=>spec.name,'number'=>0,'status'=>:error,'out'=>"Setup failed",'err'=>"",'duration'=>0)
+          end
+          @context['rutema_status']=status
+          if @teardown
+            message(:test=>spec.name,:text=>'teardown')
+            executed_steps,teardown_status = run_scenario("_teardown_", @teardown.scenario, @context, true)
+            status=teardown_status unless STATUS_CODES.find_index(teardown_status) < STATUS_CODES.find_index(status)
+          end
+          @context['rutema_status']=status
+          message(:test=>spec.name,:text=>'finished')
+          state['status']=status
+          state["stop_time"]=Time.now
+          state['steps']=steps
+          @number_of_runs+=1
+          return state
+        ensure
+          begin
+            cleanup_exception = nil
+            @cleanup_blocks.each do |cleanup_block|
+              # Try all blocks
+              begin
+                cleanup_block.run(@context) if cleanup_block.respond_to?(:run)
+              rescue Exception => e
+                # Ignore errors, ensure all cleanup steps are attempted
+                cleanup_exception = e
+              end
+            end
+            raise cleanup_exception if !cleanup_exception.nil?
+          ensure
+            @cleanup_blocks = []
+          end
         end
-        message(:test=>spec.name,:text=>'finished')
-        state["stop_time"]=Time.now
-        state['steps']=steps
-        @number_of_runs+=1
-        return state
       end
 
       private
-      def run_scenario name,scenario,meta
+
+      def run_scenario(name, scenario, meta, is_special)
         executed_steps=[]
-        status=:warning
+        status=:skipped
         begin 
           stps=scenario.steps
           if stps.empty?
@@ -56,18 +83,45 @@ module Rutema
             status=:error
           else
             stps.each do |s|
-              executed_steps<<run_step(s,meta)
-              message(:test=>name,:text=>s.to_s,'number'=>s.number,'status'=>s.status,'out'=>s.output,'err'=>s.error,'duration'=>s.exec_time)
-              status=s.status
-              break if :error==s.status
+              if status == :error && s.skip_on_error?
+                message(:test=>name,:text=>s.to_s,'number'=>s.number,'status'=>:skipped,'is_special'=>is_special)
+              else
+                message(
+                  :test => name, :text => s.to_s, 'number' => s.number,
+                  'status' => :started, 'is_special' => is_special
+                )
+                sleep 0.05
+                begin
+                  cache_cleanup(s)
+                  executed_steps << run_step(s, meta)
+                rescue Exception => e
+                  throw e unless s.continue?
+                  s.status = :error
+                end
+                message(
+                  :test => name, :text => s.to_s, 'number' => s.number,
+                  'status' => s.status, 'out' => s.output, 'err' => s.error,
+                  'backtrace' => s.backtrace, 'duration' => s.exec_time,
+                  'is_special' => is_special
+                )
+                status=s.status unless STATUS_CODES.find_index(s.status) < STATUS_CODES.find_index(status)
+                break if :error == s.status and !s.continue?
+              end
             end
           end
-        rescue  
+        rescue
           error(name,$!.message)
           status=:error
         end
         return executed_steps,status
       end
+
+      def cache_cleanup(step)
+        if step.has_cleanup? && step.cleanup.respond_to?(:run)
+          @cleanup_blocks << step.cleanup
+        end
+      end
+
       def run_step step,meta
         if step.has_cmd? && step.cmd.respond_to?(:run)
           step.cmd.run(meta)
@@ -80,7 +134,21 @@ module Rutema
       end
     end
 
+    ##
+    # Fake runner which does not run the passed steps but just sets their
+    # execution status to +:success+
+    #
+    # Steps that do not respond to +:run+ have their status set to +:warning+.
+    #
+    # Returns the step after "executing" it successfully
     class NoOp<Default
+      ##
+      # Simulate running the step by setting its status to +:success+
+      #
+      # If the step does not respond to +:run+ then +:warning+ is set as its
+      # status.
+      #
+      # * +step+ - 
       def run_step step,meta
         unless step.has_cmd? && step.cmd.respond_to?(:run)
           message("No command associated with step '#{step.step_type}'. Step number is #{step.number}")
@@ -91,4 +159,4 @@ module Rutema
       end
     end
   end
-end
+end

data/lib/rutema/elements/minimal.rb

@@ -1,14 +1,15 @@
-#  Copyright (c) 2007-2017 Vassilis Rizopoulos. All rights reserved.
+#  Copyright (c) 2007-2021 Vassilis Rizopoulos. All rights reserved.
+
 require 'highline'
+
 module Rutema
-  #The Elements module provides the namespace for the various modules adding parser functionality
+  ##
+  # Module providing a namespace for modules which are used to add to a parsers
+  # functionality which can then be utilized in test specifications
   module Elements
-    #Minimal offers a minimal(chic) set of elements for use in specifications
-    #
-    #These are:
-    # echo
-    # command
-    # prompt
+    ##
+    # Module offering an examplary minimal set of elements for use as steps in
+    # test specifications
     module Minimal
       #echo prints a message on the screen:
       # <echo text="A meaningful message"/>
@@ -17,6 +18,7 @@ module Rutema
         step.cmd=Patir::RubyCommand.new("echo"){|cmd| cmd.error="";cmd.output="#{step.text}";$stdout.puts(cmd.output) ;:success}
         return step
       end
+
       #prompt asks the user a yes/no question. Answering yes means the step is succesful.
       # <prompt text="Do you want fries with that?"/>
       #
@@ -34,6 +36,7 @@ module Rutema
         end#do rubycommand
         return step
       end
+
       #command executes a shell command
       # <command cmd="useful_command.exe with parameters", working_directory="some/directory"/>
       def element_command step
@@ -45,4 +48,4 @@ module Rutema
       end
     end
   end
-end
+end

data/lib/rutema/parsers/xml.rb

@@ -1,4 +1,5 @@
-#  Copyright (c) 2007-2017 Vassilis Rizopoulos. All rights reserved.
+#  Copyright (c) 2007-2021 Vassilis Rizopoulos. All rights reserved.
+
 require 'rexml/document'
 require 'patir/command'
 require_relative '../core/parser'
@@ -7,14 +8,17 @@ require_relative '../elements/minimal'
 
 module Rutema
   module Parsers
-    #Rutema::Parsers::XML is a basic XML parser that is easily extended
+    ##
+    # Basic and easily extendable parser for test specifications in XML format
     #
-    #Derive your parser from this class and define for each element 'foo' that you want to parse 
-    #a method element_foo(step)
+    # Actual XML specification parsers should be derived from this class and
+    # define for each specification element +foo+ to be parsed a method
+    # +element_foo+.
     #
-    #The method will receive a Rutema::Step instance as a parameter which it should return
+    # The method will receive a Rutema::Step instance as a parameter which it should return
     class XML<SpecificationParser
       include Rutema::Elements::Minimal
+
       #:nodoc:
       ELEM_SPEC="specification"
       #:nodoc:
@@ -23,11 +27,15 @@ module Rutema
       ELEM_TITLE="specification/title"
       #:nodoc:
       ELEM_SCENARIO="specification/scenario"
-      #Parses __param__ and returns the Rutema::Specification instance
+
+      ##
+      # Pass the given test specification and return a corresponding
+      # Specification instance
       #
-      #param can be the filename of the specification or the contents of that file.
+      # The passed argument can either be the path to a test specification file
+      # or the test specification itself.
       #
-      #Will throw Rutema::ParserError if something goes wrong
+      # This will raise ParserError if an error occurs during parsing.
       def parse_specification param
         @parsed||=[]
         begin
@@ -47,8 +55,17 @@ module Rutema
           raise Rutema::ParserError,$!.message
         end
       end
+
       private
-      #Parses the XML specification of a testcase and creates the corresponding Rutema::Specification instance
+
+      ##
+      # Parse the XML specification of a testcase and create a corresponding
+      # Rutema::Specification instance
+      #
+      # * +xmltext+ - the actual test specification text which must be an XML
+      #   document
+      # * +filename+ - the filename of the test specification file or the
+      #   current working directory of the test execution (this 
       def parse_case xmltxt,filename
         spec=Rutema::Specification.new({})
         xmldoc=REXML::Document.new( xmltxt )
@@ -71,12 +88,21 @@ module Rutema
         spec.filename=filename
         return spec
       end
-      #Validates the XML file from our point of view.
+
+      ##
+      # Conduct a simple validation of the XML document by checking if it
+      # contains all necessary elements
+      #
+      # ParserError is being raised if any of the necessary elements is missing.
+      #
+      # * +xmldoc+ - the text of the XML document to be checked for the
+      # necessary elements
       def validate_case xmldoc
         raise Rutema::ParserError,"missing #{ELEM_SPEC} element in #{xmldoc}" unless xmldoc.elements[ELEM_SPEC]
         raise Rutema::ParserError,"missing #{ELEM_DESC} element in #{xmldoc}" unless xmldoc.elements[ELEM_DESC]
         raise Rutema::ParserError,"missing #{ELEM_TITLE} element in #{xmldoc}" unless xmldoc.elements[ELEM_TITLE]
       end
+
       #Parses the 'scenario' XML element and returns the Rutema::Scenario instance
       def parse_scenario xmltxt
         scenario=Rutema::Scenario.new([])
@@ -103,12 +129,14 @@ module Rutema
         end
         return scenario
       end
+
       #Parses xml and returns the Rutema::Step instance
       def parse_step xmltxt
         xmldoc=REXML::Document.new( xmltxt )
         #any step element
         step=Rutema::Step.new()
         step.ignore=false
+        step.continue=false
         xmldoc.root.attributes.each do |attr,value|
          add_attribute(step,attr,value)
         end
@@ -116,17 +144,38 @@ module Rutema
         step.step_type=xmldoc.root.name
         return step
       end
+
+      ##
+      # Add an attribute of a given name with the given value to a specification
+      # element
+      #
+      # * +element+ - the specification element the attribute shall be added to
+      # * +attr+ - the name of the attribute which shall either be created or
+      #   whose current value will be overridden
+      # * +value+ - the value which shall be set for the attribute
       def add_attribute element,attr,value
+        # If the string is a textual representation of a boolean value ...
         if boolean?(value)
+          # ... convert it to a boolean value
          element.attribute(attr,eval(value))
         else
           element.attribute(attr,value)
         end
       end
+
+      ##
+      # Check if attribute_value is a string representing a boolean value
+      #
+      # This returns +true+ if the string is "true" or "false" or +false+
+      # otherwise.
+      #
+      # * +attribute_value+ - the entity which shall be checked if it's a string
+      # representing a boolean value
       def boolean? attribute_value
         return true if attribute_value=="true" || attribute_value=="false"
         return false
       end
+
       #handles <include_scenario> elements, adding the steps to the current scenario
       def include_scenario step
         raise Rutema::ParserError,"missing required attribute file in #{step}" unless step.has_file?
@@ -135,6 +184,9 @@ module Rutema
         include_content=File.read(step.file)
         return parse_scenario(include_content)
       end
+
+      ##
+      # 
       def extension_handling spec
         #change into the directory the spec is in to handle relative paths correctly
         Dir.chdir(File.dirname(File.expand_path(spec.filename))) do |path|
@@ -144,7 +196,7 @@ module Rutema
               begin
                 self.send(:"element_#{step.step_type}",step)
               rescue
-                raise ParserError, $!.message
+                raise ParserError, ($!.message + "\n" + $@.join("\n"))
               end#begin
             elsif @configuration.parser["strict_mode"]
               raise ParserError,"No command element associated with #{step.step_type}. Missing element_#{step.step_type}"

data/lib/rutema/reporters/json.rb

@@ -1,4 +1,5 @@
-#  Copyright (c) 2007-2010 Vassilis Rizopoulos. All rights reserved.
+#  Copyright (c) 2007-2021 Vassilis Rizopoulos. All rights reserved.
+
 require 'json'
 require_relative "../core/reporter"
 
@@ -12,11 +13,12 @@ module Rutema
     class JSON<Rutema::Reporters::BlockReporter
       #Default report filename
       DEFAULT_FILENAME="rutema.results.json"
-    
+
       def initialize configuration,dispatcher
         super(configuration,dispatcher)
         @filename=configuration.reporters.fetch(self.class,{}).fetch("filename",DEFAULT_FILENAME)
       end
+
       #We get all the data from a test run in here.
       def report specs,states,errors
         run_entry={}

data/lib/rutema/reporters/junit.rb

@@ -1,4 +1,5 @@
-#  Copyright (c) 2015 Vassilis Rizopoulos. All rights reserved.
+#  Copyright (c) 2015-2021 Vassilis Rizopoulos. All rights reserved.
+
 require 'rexml/document'
 require_relative "../core/reporter"
 
@@ -18,16 +19,18 @@ module Rutema
     # cfg.reporter={:class=>Rutema::Reporters::JUnit,"filename"=>"rutema.junit.xml"}
     class JUnit<BlockReporter
       DEFAULT_FILENAME="rutema.results.junit.xml"
-    
+
       def initialize configuration,dispatcher
         super(configuration,dispatcher)
         @filename=configuration.reporters.fetch(self.class,{}).fetch("filename",DEFAULT_FILENAME)
       end
+
       #We get all the data from a test run in here.
       def report specs,states,errors
         cnt=process_data(specs,states,errors)
         Rutema::Utilities.write_file(@filename,cnt)
       end
+
       def process_data specs,states,errors
         tests=[]
         number_of_failed=0
@@ -49,7 +52,9 @@ module Rutema
         }
         return junit_content(tests,attributes,errors)
       end
+
       private
+
       def test_case name,state
         #<testcase name="" time="">      => the results from executing a test method
         #  <system-out>  => data written to System.out during the test run
@@ -61,19 +66,35 @@ module Rutema
         element_test=REXML::Element.new("testcase")
         element_test.add_attributes("name"=>name,"time"=>state.duration,"classname"=>@configuration.context[:config_name])
         if state.status!=:success
-          fail=REXML::Element.new("failure")
-          fail.add_attribute("message","Step #{state.steps.last.number} failed.")
-          fail.add_text "Step #{state.steps.last.number} failed."
-          element_test.add_element(fail)
-          out=REXML::Element.new("system-out")
-          out.add_text state.steps.last.out
-          element_test.add_element(out)
-          err=REXML::Element.new("system-err")
-          err.add_text state.steps.last.err
-          element_test.add_element(err)
+          failed_steps = state.steps.select {|step| !step.status.nil? && STATUS_CODES.find_index(:success) < STATUS_CODES.find_index(step.status)}
+          if !failed_steps.empty?
+            failed_steps.each do |step|
+              fail=REXML::Element.new("failure")
+              fail.add_attribute("message","Step: #{step.text} reported non-success status: #{step.status}.")
+              fail.add_text "Step #{step.number} failed."
+              element_test.add_element(fail)
+              out=REXML::Element.new("system-out")
+              out.add_text step.out
+              element_test.add_element(out)
+              err=REXML::Element.new("system-err")
+              err.add_text step.err
+              element_test.add_element(err)
+            end
+          else
+            fail.add_attribute("message","Case reported non-success status: #{state.status} without a matching step state.")
+            fail.add_text "Case with #{state.steps.last.number} steps failed with no matching failed step."
+            element_test.add_element(fail)
+            out=REXML::Element.new("system-out")
+            out.add_text state.steps.last.out
+            element_test.add_element(out)
+            err=REXML::Element.new("system-err")
+            err.add_text state.steps.last.err
+            element_test.add_element(err)
+          end
         end
         return element_test
       end
+
       def crash name,message
         failed=REXML::Element.new("testcase")
         failed.add_attributes("name"=>name,"classname"=>@configuration.context[:config_name],"time"=>0)
@@ -83,6 +104,7 @@ module Rutema
         failed.add_element(msg)
         return failed
       end
+
       def junit_content tests,attributes,errors
         element_suite=REXML::Element.new("testsuite")
         element_suite.add_attributes(attributes)        
@@ -90,6 +112,7 @@ module Rutema
         tests.each{|t| element_suite.add_element(t)}
         return document(element_suite).to_s
       end
+
       def document suite
         xmldoc=REXML::Document.new
         xmldoc<<REXML::XMLDecl.new

data/lib/rutema/version.rb

@@ -1,9 +1,24 @@
+# Copyright (c) 2021 Vassilis Rizopoulos. All rights reserved.
+
+# frozen_string_literal: true
+
+##
+# The top-level module of rutema encompassing all its functionality.
 module Rutema
-  #This module defines the version numbers for the library
+  ##
+  # Version information of the rutema gem
   module Version
-    MAJOR=2
-    MINOR=0
-    TINY=0
-    STRING=[ MAJOR, MINOR, TINY ].join( "." )
+    ##
+    # The major version of the rutema gem
+    MAJOR = 2
+    ##
+    # The minor version of the rutema gem
+    MINOR = 0
+    ##
+    # The tiny version of the rutema gem
+    TINY = 1
+    ##
+    # The version information of the rutema gem as a string
+    STRING = [MAJOR, MINOR, TINY].join(".")
   end
-end
+end