rutema 2.0.0 → 2.0.1

data/lib/rutema/core/engine.rb

@@ -1,4 +1,5 @@
-#  Copyright (c) 2007-2017 Vassilis Rizopoulos. All rights reserved.
+#  Copyright (c) 2007-2021 Vassilis Rizopoulos. All rights reserved.
+
 require 'thread'
 require_relative 'parser'
 require_relative 'reporter'
@@ -6,13 +7,28 @@ require_relative 'runner'
 require_relative '../version'
 
 module Rutema
-  #Rutema::Engine implements the rutema workflow.
+  ##
+  # Class implementing the inner workflow of rutema
   #
-  #It instantiates the configured parser, runner and reporter instances and wires them together via Rutema::Dispatcher
+  # This class takes care of instantiating the configured parser, runner and
+  # reporters. The reporters then receive event information from the runner
+  # through a Dispatcher instance.
   #
-  #The full workflow is Parse->Run->Report and corresponds to one call of the Engine#run method
+  # The full workflow consists of subsequent +parse+, +run+ and +report+ phases
+  # and corresponds to one call of the #run method.
   class Engine
     include Messaging
+
+    ##
+    # Initialize a new Engine instance and setup all class instances needed for
+    # test execution
+    #
+    # This brings up a parser, the runner and the reporters and wires them up as
+    # needed. After completion of this method the instance is ready for test
+    # execution by means of the #run method.
+    #
+    # * +configuration+ - a Configuration instance according to which Engine,
+    #   its components and the test run shall be set up
     def initialize configuration
       @queue=Queue.new
       @parser=instantiate_class(configuration.parser,configuration) if configuration.parser
@@ -29,27 +45,38 @@ module Rutema
       @dispatcher=Dispatcher.new(@queue,configuration)
       @configuration=configuration
     end
-    #Parse, run, report
+
+    ##
+    # Parse the test specifications, execute the test(s) and report the results
+    #
+    # * +test_identifier+ - an optional identifier of a single test case to be
+    #   executed, this cannot be an arbitrary one but must still be contained
+    #   in the configured test specification identifiers
     def run test_identifier=nil
       @dispatcher.run!
-      #start
+      # start
       message("start")
       suite_setup,suite_teardown,setup,teardown,tests=*parse(test_identifier)
       if tests.empty?  
         @dispatcher.exit
         raise RutemaError,"No tests to run!"
       else
-        @runner.setup=setup
-        @runner.teardown=teardown
-        #running - at this point we've done any and all checks and we're stepping on the gas
+        # running - at this point all checks are done and the tests are active
         message("running")
-        run_scenarios(tests,suite_setup,suite_teardown)
+        run_scenarios(tests,suite_setup,suite_teardown,setup,teardown)
       end
       message("end")
       @dispatcher.exit
       @dispatcher.report(tests)
     end
-    #Parse a single test spec or all the specs listed in the configuration
+
+    ##
+    # Parse a single test specification or all the specs given by the
+    # configuration
+    #
+    # * +test_identifier+ - an optional identifier of a single test case to be
+    #   executed, this cannot be an arbitrary one but must still be contained
+    #   in the configured test specification identifiers
     def parse test_identifier=nil
       specs=[]
       #so, while we are parsing, we have a list of tests
@@ -68,12 +95,27 @@ module Rutema
       suite_setup,suite_teardown,setup,teardown=parse_specials(@configuration)
       return [suite_setup,suite_teardown,setup,teardown,specs]
     end
+
     private
+
+    ##
+    # Parse an array of test specifications into Specification instances
+    #
+    # * +tests+ - an array containing paths to test specification files or the
+    #   test specifications' texts themselves
     def parse_specifications tests
       tests.map do |t| 
         parse_specification(t)
       end.compact
     end
+
+    ##
+    # Parse a single test specification into a Specification instance
+    #
+    # * +spec_identifier+ - either the path to a test specification file or the
+    #   actual test specification text itself
+    #
+    # A ParserError is raised upon failure.
     def parse_specification spec_identifier
       begin
         @parser.parse_specification(spec_identifier)
@@ -82,6 +124,15 @@ module Rutema
         raise Rutema::ParserError, "In #{spec_identifier}: #{$!.message}" 
       end
     end
+
+    ##
+    # Parse the special test (suite) setup and teardown methods if set
+    #
+    # This returns an array containing Specification instances for
+    # * test suite setup
+    # * test suite teardown
+    # * test setup
+    # * test teardown
     def parse_specials configuration
       suite_setup=nil
       suite_teardown=nil
@@ -101,33 +152,55 @@ module Rutema
       end
       return suite_setup,suite_teardown,setup,teardown
     end
-    def run_scenarios specs,suite_setup,suite_teardown
+
+    ##
+    # Execute the passed test Specification instances
+    #
+    # * +specs+ - an array of Specification instances representing the actual
+    #   tests
+    # * +suite_setup+ - a test suite setup Specification instance
+    # * +suite_teardown+ - a test suite teardown Specification instance
+    def run_scenarios(specs, suite_setup, suite_teardown, setup, teardown)
       if specs.empty?
         error(nil,"No tests to run")
       else
-        if suite_setup
-          if run_test(suite_setup)==:success
-            specs.each{|s| run_test(s)}
-          else
-            error(nil,"Suite setup test failed")
-          end
-        else
+        @runner.setup = nil
+        @runner.teardown = nil
+
+        if !suite_setup || (run_test(suite_setup, true) == :success)
+          @runner.setup = setup
+          @runner.teardown = teardown
           specs.each{|spec| run_test(spec)}
+        else
+          error(nil, "Suite setup test failed")
         end
         if suite_teardown
-          run_test(suite_teardown)
+          @runner.setup = nil
+          @runner.teardown = nil
+          run_test(suite_teardown, true)
         end
       end
     end
-    def run_test specification
+
+    ##
+    #
+    def run_test(specification, is_special = false)
       if specification.scenario
-        status=@runner.run(specification)["status"]
+        status = @runner.run(specification, is_special)["status"]
       else
         status=:not_executed
         message(:test=>specification.name,:text=>"No scenario", :status=>status)
       end
       return status
     end
+
+    ##
+    # Instantiate a new class of a given type passing it a given configuration
+    # upon construction
+    #
+    # * +definition+ - class of which a new instance shall be instantiated
+    # * +configuration+ - Configuration instance which shall be passed to the
+    #   initializer of the to be created instance
     def instantiate_class definition,configuration
       if definition[:class]
         klass=definition[:class]
@@ -135,26 +208,55 @@ module Rutema
       end
       return nil
     end
+
+    ##
+    # Check if the given test identifier belongs to the normal test cases or to
+    # one of the special ones (the test (suite) setups and teardowns)
+    #
+    # * +test_identifier+ - the test identifier to check against membership in
+    #   the normal or special test case identifier sets
     def is_spec_included? test_identifier
       full_path=File.expand_path(test_identifier)
-      return @configuration.tests.include?(full_path) || is_special?(test_identifier) 
+      return @configuration.tests.include?(full_path) || is_special?(test_identifier)
     end
+
+    ##
+    # Check if the given test identifier is a (suite) setup or teardown one
+    #
+    # This checks if the given identifier was configured as (suite) setup or
+    # teardown within the rutema run's configuration.
+    #
+    # * +test_identifier+ - the test identifier which shall be checked for being
+    #   a special one
     def is_special? test_identifier
       full_path=File.expand_path(test_identifier)
       return full_path==@configuration.suite_setup ||
       full_path==@configuration.suite_teardown ||
       full_path==@configuration.setup ||
-      full_path==@configuration.teardown 
+      full_path==@configuration.teardown
     end
   end
-  #The Rutema::Dispatcher functions as a demultiplexer between Rutema::Engine and the various reporters.
-  #
-  #In stream mode the incoming queue is popped periodically and the messages are destributed to the queues of any subscribed event reporters.
+
+  ##
+  # Class functioning as a demultiplexer between the Engine and the various
+  # Reporters instances
   #
-  #By default this includes Rutema::Reporters::Collector which is then used at the end of a run to provide the collected data to all registered block mode reporters 
+  # In stream mode the incoming queue is popped periodically and the messages
+  # are distributed to the queues of any subscribed event reporters. By default
+  # this includes Reporters::Collector which is then used at the end of a run to
+  # provide the collected data to all registered block mode reporters 
   class Dispatcher
-    #The interval between queue operations
-    INTERVAL=0.01
+    ##
+    # The interval between queue operations
+    INTERVAL = 0.01
+
+    ##
+    # Initialize a new demultiplexer and instantiate all reporters requested by
+    # the passed configuration
+    #
+    # * +queue+ - the queue which will be shared between the Engine instance and
+    #   the Reporter instances
+    # * +configuration+ - the Configuration instance of the rutema run
     def initialize queue,configuration
       @queue = queue
       @queues = {}
@@ -169,12 +271,25 @@ module Rutema
       @streaming_reporters<<@collector
       @configuration=configuration
     end
-    #Call this to establish a queue with the given identifier
+
+    ##
+    # Call this to establish a queue with the given identifier
+    #
+    # This method will create a new queue within the Dispatcher instance into
+    # which data from the incoming queue from the Engine instance will be
+    # dispatched to.
+    #
+    # * +identifier+ - a unique identifier for the queue. If two identifiers
+    #   collide the new subscriber will replace the earlier one
     def subscribe identifier
       @queues[identifier]=Queue.new
       return @queues[identifier]
     end
-    
+
+    ##
+    # Start #update threads of all event/streaming reporters and then start a
+    # new locally managed thread which continually dispatches messages from the
+    # incoming queue
     def run!
       puts "Running #{@streaming_reporters.size} streaming reporters" if $DEBUG
       @streaming_reporters.each {|r| r.run!}
@@ -186,12 +301,19 @@ module Rutema
       end
     end
 
+    ##
+    # Call all block reporters' BlockReporter#report method
     def report specs
       @block_reporters.each do |r|
         r.report(specs,@collector.states,@collector.errors)
       end
       Reporters::Summary.new(@configuration,self).report(specs,@collector.states,@collector.errors)
     end
+
+    ##
+    # Dispatch all messages in the incoming queue to the subscribed reporters,
+    # exit all streaming reporters' threads and then stop the own internal
+    # dispatch thread
     def exit
       puts "Exiting main dispatcher" if $DEBUG
       if @thread
@@ -200,7 +322,12 @@ module Rutema
         Thread.kill(@thread)
       end
     end
+
     private
+
+    ##
+    # Dispatch messages from the incoming queue to all subscribers until the
+    # incoming queue is empty
     def flush
       puts "Flushing queues" if $DEBUG
       if @thread
@@ -210,6 +337,18 @@ module Rutema
         end
       end
     end
+
+    ##
+    # Instantiate a new reporter instance and pass the configuration and this
+    # Dispatcher instance this method is called upon to
+    #
+    # * +definition+ - hash containing the class type which shall be
+    #   instantiated referenced by its +:class+ key
+    # * +configuration+ - the Configuration instance which shall be passed to
+    #   the initializer of the to be instantiated class
+    #
+    # This either returns the new class instance or _nil_ if the passed hash
+    # did not contain a key +:class+
     def instantiate_reporter definition,configuration
       if definition[:class]
         klass=definition[:class]
@@ -217,6 +356,11 @@ module Rutema
       end
       return nil
     end
+
+    ##
+    # Pop the last element of the incoming queue from the runner (if it's not
+    # empty) and distribute it to all subscribed Reporters::EventReporter
+    # instances
     def dispatch
       if @queue.size>0
         data=@queue.pop
@@ -224,4 +368,4 @@ module Rutema
       end
     end
   end
-end
+end

data/lib/rutema/core/framework.rb

@@ -1,13 +1,37 @@
+#  Copyright (c) 2021 Vassilis Rizopoulos. All rights reserved.
+
 module Rutema
-  #Represents the data beeing shunted between the components in lieu of logging.
+  STATUS_CODES=[:started,:skipped,:success,:warning,:error]
+
+  ##
+  # Simple base for classes concerned with message passing to report test
+  # progress and failures
+  #
+  # This class and its descendants can be utilized as a container for data
+  # relevant to tests and their results. Currently they are being emitted by the
+  # Engine and Runners instances and consumed by classes within the  Reporters
+  # module.
   #
-  #This is the primary type passed to the event reporters
+  # Specialized descendants are ErrorMessage and RunnerMessage.
   class Message
-    attr_accessor :test,:text,:timestamp
-    #Keys used:
-    # test - the test id/name
-    # text - the text of the message
-    # timestamp
+    ##
+    # The test whose execution originated the message
+    attr_accessor :test
+    ##
+    # The text of the message
+    attr_accessor :text
+    ##
+    # The timestamp of the message's creation
+    attr_accessor :timestamp
+
+    ##
+    # Initialize a new message from data passed in a hash
+    #
+    # The following keys of the hash are being utilized:
+    # * +:test+ - the test id/name of the test which originates the message
+    # * +:text+ - the text of the message
+    # * +:timestamp+ - most often the timestamp of the creation of the message,
+    #   defaults to +Time.now+
     def initialize params
       @test=params.fetch(:test,"")
       @test||=""
@@ -15,6 +39,8 @@ module Rutema
       @timestamp=params.fetch(:timestamp,Time.now)
     end
 
+    ##
+    # Convert the instance to a convenient textual representation
     def to_s
       msg=""
       msg<<"#{@test} " unless @test.empty?
@@ -22,8 +48,16 @@ module Rutema
       return msg
     end
   end
-  #What it says on the tin.
+
+  ##
+  # Message container to report test errors
+  #
+  # The reported on errors may concern the test specifications, parser errors or
+  # errors which occurred during test execution. Logic errors of rutema itself
+  # are not reported by means of this class.
   class ErrorMessage<Message
+    ##
+    # Convert the instance to a convenient textual representation
     def to_s
       msg="ERROR - "
       msg<<"#{@test} " unless @test.empty?
@@ -31,12 +65,26 @@ module Rutema
       return msg
     end
   end
-  #The Runner continuously sends these when executing tests
+
+  ##
+  # Message container continually being emitted by runners (see Runners module)
+  # during test execution
   #
-  #If there is an engine error (e.g. when parsing) you will get an ErrorMessage, if it is a test error
-  #you will get a RunnerMessage with :error in the status.
+  # These messages inform about the progress of test execution. Test errors are
+  # propagated through instances of this class as well. If it's an engine error
+  # (e.g. during parsing), then an ErrorMessage will be used in that case.
   class RunnerMessage<Message
-    attr_accessor :duration,:status,:number,:out,:err
+    attr_accessor :duration, :status, :number, :out, :err, :is_special
+
+    ##
+    # Initialize a new runner message from data passed in a hash
+    #
+    # Additionally to the keys of the Message initializer the following keys of
+    # the hash are being utilized:
+    # * "duration" - the time a test step took for execution
+    # * "status" - the status of the respective step
+    # * +:timestamp+ - most often the timestamp of the creation of the message,
+    #   defaults to +Time.now+
     def initialize params
       super(params)
       @duration=params.fetch("duration",0)
@@ -44,13 +92,18 @@ module Rutema
       @number=params.fetch("number",1)
       @out=params.fetch("out","")
       @err=params.fetch("err","")
+      @backtrace=params.fetch("backtrace","")
+      @is_special=params.fetch("is_special","")
     end
 
+    ##
+    # Convert the instance to a convenient textual representation
     def to_s
       msg="#{@test}:"
+      msg<<" #{@timestamp.strftime("%H:%M:%S")} :"
       msg<<"#{@text}." unless @text.empty?
       outpt=output()
-      msg<<" Output:\n#{outpt}" unless outpt.empty? || @status!=:error
+      msg<<" Output" + (outpt.empty? ? "." : ":\n#{outpt}") # unless outpt.empty? || @status!=:error
       return msg
     end
 
@@ -58,9 +111,11 @@ module Rutema
       msg=""
       msg<<"#{@out}\n" unless @out.empty?
       msg<<@err unless @err.empty?
+      msg<<"\n" + (@backtrace.kind_of?(Array) ? @backtrace.join("\n") : @backtrace) unless @backtrace.empty?
       return msg.chomp
     end
   end
+
   #While executing tests the state of each test is collected in an 
   #instance of ReportState and the collection is at the end passed to the available block reporters
   #
@@ -68,7 +123,7 @@ module Rutema
   #and accumulates the duration reported by all messages in it's collection.
   class ReportState
     attr_accessor :steps
-    attr_reader :test,:timestamp,:duration,:status
+    attr_reader :test, :timestamp, :duration, :status, :is_special
     
     def initialize message
       @test=message.test
@@ -76,25 +131,43 @@ module Rutema
       @duration=message.duration
       @status=message.status
       @steps=[message]
+      @is_special=message.is_special
     end
 
     def <<(message)
       @steps<<message
       @duration+=message.duration
-      @status=message.status
+      @status = message.status unless message.status.nil? \
+        || (!@status.nil? && STATUS_CODES.find_index(message.status) < STATUS_CODES.find_index(@status))
     end
   end
 
+  ##
+  # Mix-in module which offers an interface to push messages to a queue
+  #
+  # Instances of the class including this module need a @queue member variable.
   module Messaging
-    #Signal an error - use the test name/id as the identifier
+    ##
+    # Push a new ErrorMessage instance to the queue
+    #
+    # * +identifier+ - in most cases this would be the name of a test or its
+    #   specification file
+    # * +message+ - a short descriptive message detailing the error condition
     def error identifier,message
       @queue.push(ErrorMessage.new(:test=>identifier,:text=>message,:timestamp=>Time.now))
     end
-    #Informational message during test runs
+
+    ##
+    # Push a new Message or RunnerMessage instance to the queue
+    #
+    # If +message+ is of type String a Message instance will be pushed to the
+    # queue. If it's of type Hash it will be passed to the initializer of
+    # RunnerMessage if it has both the keys :test and "status" or to the
+    # initializer of Message if not so.
     def message message
       case message
       when String
-        Message.new(:text=>message,:timestamp=>Time.now)
+        @queue.push(Message.new(:text=>message,:timestamp=>Time.now))
       when Hash
         hm=Message.new(message)
         hm=RunnerMessage.new(message) if message[:test] && message["status"]
@@ -103,16 +176,32 @@ module Rutema
       end
     end
   end
-  #Generic error class for errors in the engine
+
+  ##
+  # Generic error class which is being used as base class for all other rutema
+  # errors and for Engine related errors.
+  #
+  # This is being inherited by:
+  # * ParserError
+  # * ReportError
+  # * RunnerError
   class RutemaError<RuntimeError
   end
-  #Is raised when an error is found in a specification
-  class ParserError<RutemaError
+
+  ##
+  # Specialized error class particular to the parsing of rutema test
+  # specifications
+  class ParserError < RutemaError
   end
-  #Is raised on an unexpected error during execution
-  class RunnerError<RutemaError
+
+  ##
+  # Specialized error class designated to errors within runner classes
+  class RunnerError < RutemaError
   end
-  #Errors in reporters should use this class
-  class ReportError<RutemaError
+
+  ##
+  # Specialized error class which should be utilized by Reporters members to
+  # signal errors upon reporting
+  class ReportError < RutemaError
   end
-end
+end