framework_gem 0.1.0

data/lib/lib/test_case/test_error.rb

@@ -0,0 +1,29 @@
+require './lib/test_case/test_trace'
+
+class TestError < TestTrace
+
+  attr_reader(:message, :step)
+
+  # Public: Initializes a new Error.
+  #
+  # test_case - TestCase instance in which the error occurred.
+  # message   - String error message.
+  # backtrace - Array containing error backtrace.
+  # step      - TestStep in which error occurred (default: nil).
+  #
+  # Returns new Error instance.
+  def initialize(test_case, message, backtrace, step: nil)
+    super(test_case, backtrace)
+    @message = message
+    @step = step
+    @hash.update(message: @message, step_id: @step.nil? ? nil : @step.id, step_name: @step.nil? ? nil : @step.name)
+  end
+
+  # Public: Gets string representation of error.
+  #
+  # Returns error String.
+  def to_s
+    error_hash = {file: @file, line: @line, method: @method, line_in_method: @line_in_method}
+    "#<Error #{error_hash.map {|k, v| "#{k}=#{v}"}.join(', ')}>"
+  end
+end

data/lib/lib/test_case/test_exception.rb

@@ -0,0 +1,9 @@
+
+# The TestException is raised on any failure
+
+class TestException < Exception
+
+  def initialize
+    super
+  end
+end

data/lib/lib/test_case/test_status.rb

@@ -0,0 +1,39 @@
+
+# The TestStatus module defines all the possible statuses for tests and steps
+
+class TestStatus
+  NOT_RUN = :'not run'
+  RUNNING = :running
+  PASSED  = :passed
+  DONE    = :done     # currently, performance tests do not pass or fail in TMC--this is determined by SWORD
+  FAILED  = :failed
+  ERROR   = :error
+
+  attr_reader(:status, :reason, :backtrace, :trace)
+
+  def initialize(test_case, status, reason: nil, backtrace: nil, exception: nil, trace: nil)
+    @status = status
+    @backtrace = exception.nil? ? backtrace : exception.backtrace
+    if !exception.nil?
+      if (exception.message || '').empty?
+        @reason = exception.class.name
+      else
+        @reason = exception.message
+      end
+    else
+      @reason = reason
+    end
+    if !@backtrace.nil?
+      @trace = TestTrace.new(test_case, @backtrace)
+    else
+      @trace = trace
+    end
+  end
+
+  def to_h
+    {
+      status: @status.to_s,
+      reason: @reason
+    }.merge(@trace.nil? ? {} : @trace.to_h)
+  end
+end

data/lib/lib/test_case/test_step.rb

@@ -0,0 +1,156 @@
+require './lib/test_case/test_status'
+
+# The TestStep class defines test step functionality, managing flow and providing hooks
+
+class TestStep
+  attr_reader(:id, :name, :device, :iteration)
+
+  # Public: Creates a new test step.
+  #
+  # test_case     - TestCase (subclass) instance.
+  # id            - Integer test step number.
+  # name          - String test step name.
+  # device        - Integer test device id.
+  # iteration     - Integer test iteration number.
+  # status        - TestStatus initial status (default: nil).
+  # ignore_errors - Boolean indicating the step should ignore any errors (default: nil).
+  #                 If default, setting from test case will be used.
+  #
+  # Returns the new TestStep instance.
+  def initialize(test_case, id, name, device, iteration, status: nil, ignore_errors: nil)
+    @test_case = test_case
+    @id = id
+    @name = name
+    @device = device
+    @iteration = iteration
+    @status = TestStatus.new(@test_case, status)
+    @update = false
+    @raising = !(ignore_errors.nil? ? @test_case.test_steps_ignore_errors? : ignore_errors)
+  end
+
+  # Public: Gets step status.
+  #
+  # Returns String status.
+  def status
+    @status.status
+  end
+
+  # Public: Performs the test step.
+  #
+  # block - Block that performs the step.
+  #
+  # Returns nothing.
+  def perform(update: false, &block)
+    @update = update
+    begin
+      before_run
+      run(&block)
+      pass unless status == TestStatus::PASSED
+    rescue Exception => e
+      if e.is_a?(TestException)
+        raise if @raising
+      elsif e.is_a?(Minitest::Assertion)
+        set_status(TestStatus.new(@test_case, TestStatus::FAILED, exception: e))
+        raise TestException.new if @raising
+      else
+        set_status(TestStatus.new(@test_case, TestStatus::ERROR, exception: e))
+        raise TestException.new if @raising || e.is_a?(Interrupt)  # ALWAYS raise Interrupt
+      end
+    ensure
+      after_run
+    end
+  end
+
+  # Public: Fails the test step.
+  #
+  # reason - String message describing the reason for failure.
+  #
+  # Returns nothing.
+  def fail(reason='Unknown cause', backtrace: nil)
+    set_status(TestStatus.new(@test_case, TestStatus::FAILED, reason: reason, backtrace: backtrace || caller))
+    raise TestException.new
+  end
+
+  # Public: Passes the test step.
+  #
+  # reason - String message describing the reason for passing.
+  #
+  # Returns nothing.
+  def pass(reason=nil)
+    set_status(TestStatus.new(@test_case, TestStatus::PASSED, reason: reason))
+  end
+
+  private
+
+  # Internal: Exposes the test case logger.
+  #
+  # Returns the TmcLogger instance.
+  def logger
+    @test_case.send(:logger_for_device, @device)
+  end
+
+  # Internal: Updates the step.
+  #
+  # Returns nothing.
+  def update
+    return unless @update
+    json = {id: @id, name: @name}
+    resp = @test_case.send(:tmc_put ,"/api/jobs/#{@test_case.job}/tests/#{@test_case.id}/steps/#{@id}", json: json)
+    @status = TestStatus.new(@test_case, resp.fetch('step', {})['status'])
+  end
+
+  # Internal: Hook that executes each time a test step status is set.
+  #
+  # Returns nothing.
+  def set_status(status, device: nil)
+    device ||= @device
+    @status = status
+    status = @status.status
+    message = @status.reason
+    msg_str = message.nil? ? '' : ": #{message}"
+    backtrace = @status.backtrace
+    trace_str = backtrace.nil? ? '' : "\n\t#{backtrace.join("\n\t")}"
+    # log status
+    error_str = ''
+    unless backtrace.nil?
+      e = @test_case.record_error(message, backtrace: backtrace)
+      error_str = "\n\t" + e.to_s
+    end
+    msg = "Test step ##{@id} #{status}#{msg_str}#{error_str}#{trace_str}"
+    if [TestStatus::FAILED, TestStatus::ERROR].include?(status)
+      logger.error(msg, tag: 'STATUS')
+    else
+      logger.info(msg, tag: 'STATUS')
+    end
+    json = @test_case.jsonify({device: device, iteration: @iteration}.merge(@status.to_h))
+    @test_case.send(:tmc_put, "/api/jobs/#{@test_case.job}/tests/#{@test_case.id}/steps/#{@id}/status", json: json)
+  end
+
+  # Internal: Hook that executes before every test step.
+  #
+  # Returns nothing.
+  def before_run
+    logger.info("Test step ##{@id}#{@name.nil? ? '' : ": #{@name}"}", screenshot: true)
+    update
+    set_status(TestStatus.new(@test_case, TestStatus::NOT_RUN)) if status.nil?
+  end
+
+  # Internal: Hook that executes after every test step.
+  #
+  # Returns nothing.
+  def after_run
+  end
+
+  # Internal: Hook that executes every test step.
+  #
+  # block - Block that performs the step.
+  #
+  # Returns nothing.
+  def run(&block)
+    set_status(TestStatus.new(@test_case, TestStatus::RUNNING))
+    if block_given?
+      block.call
+    end
+  end
+
+end

data/lib/lib/test_case/test_trace.rb

@@ -0,0 +1,80 @@
+require './lib/test_case/test_case'
+require './lib/helpers/tmc_helpers/tmc_helpers'
+
+class TestTrace
+  include TmcHelpers
+
+  attr_reader(:path, :file, :line, :method, :line_in_method)
+
+  # Public: Initializes a new test trace.
+  #
+  # test_case     - TestCase instance in which the trace occurred.
+  # backtrace     - Array containing call stack.
+  # exclude_files - Array of file names to exclude from trace (default: []).
+  #
+  # Returns new TestTrace instance.
+  def initialize(test_case, backtrace, exclude_files: [])
+    exclude_files += ['delegate.rb']
+    exclude_files.map! {|f| File.join('', f)}
+    path = backtrace.find do |t|
+      if t.include?('/ruby/gems/')
+        false
+      elsif exclude_files.any? {|f| t.include?(f)}
+        false
+      else
+        true
+      end
+    end || backtrace.first
+    @path = path.split(/[\\\/]/).last
+    parts = path.split(/\:(?!\/)/)
+    type_match = parts[0].match(/[\\\/](lib|test_cases)[\\\/]/)
+    unless type_match.nil?
+      @file = parts[0].split(type_match.to_s)[1]
+      if type_match[1] == 'lib'
+        rel_parts = parts[0].split(/[\\\/]/)
+        if rel_parts.include?('prereqs')
+          @type = 'prereq'
+          @prereq_type = rel_parts.last.match(/prereqs_for_(.*)\./)[1]
+        elsif rel_parts.include?('helpers')
+          @type = 'helper'
+        else
+          @type = 'method'
+        end
+      elsif type_match[1] == 'test_cases'
+        @type = 'script'
+      end
+    end
+    @line = parts[1].to_i
+    @method = parts[2].match(/`([^']+)'/)[1].to_s.split(' ').last
+    # best effort to get line in method
+    @line_in_method = nil
+    cls = Object.const_get(to_pascal_case(File.split(@file).last.split('.').first)) rescue nil  # try to get class from source file
+    if cls.nil?
+      # try getting the method from the test case instance
+      meth = test_case.method(@method.to_sym) rescue nil
+    else
+      # try getting the method from the class in the source file
+      meth = cls.instance_method(@method.to_sym) rescue nil
+    end
+    if !meth.nil? && !meth.source_location.nil?
+      # we found the method
+      source_line = meth.source_location[1]
+      @line_in_method = line - source_line
+    end
+    @hash = {
+      file: @file,
+      line: @line,
+      method: @method,
+      line_in_method: @line_in_method,
+      type: @type,
+      prereq_type: @prereq_type
+    }
+  end
+
+  # Public: Gets hash representation of trace.
+  #
+  # Returns Hash of trace attributes.
+  def to_h
+    @hash
+  end
+end

data/lib/lib/test_data.rb

@@ -0,0 +1,222 @@
+
+# The TestData class holds all the shared state for test cases
+
+class TestData
+
+  RW_DELAY_MS = 250
+
+  # Public: Initializes a TestData object.
+  #
+  # test_case - TestCase (subclass) instance.
+  # job       - Integer or String job (default: nil).
+  # data      - String name of file to use for test data (default: nil).
+  #
+  # Returns nothing.
+  def initialize(test_case, job: nil, data: nil)
+    @seq = 0
+    @data = {}
+    @test_case = test_case
+    @job = job
+    @data_file = data
+    @promiscuous = true  #TODO: DEPRECATED
+    @w_lock_obj = "test-data-#{@job}"
+  end
+
+  # Public: DEPRECATED
+  #
+  # val - Boolean indicating whether test data should be promiscuous; if true, test data will be refreshed on every
+  #       access.
+  #
+  # Returns nothing.
+  def promiscuous=(val)
+    unless val == true || val == false
+      raise "Invalid promiscuous value: #{val}"
+    end
+    @promiscuous = val
+  end
+
+  # Public: DEPRECATED
+  #
+  # Returns a Boolean true if test data is promiscuous, otherwise false.
+  def promiscuous?
+    @promiscuous
+  end
+
+  # Public: Checks if test data is empty.
+  #
+  # Returns a Boolean true if test data is empty, otherwise false.
+  def empty?
+    current_data.empty?
+  end
+
+  # Public: Digs a data attribute.
+  #
+  # keys - Splat Array of Symbol data attribute hierarchy to dig.
+  #
+  # Returns Object value, or nil.
+  def dig(*keys)
+    sym_keys = keys.map {|k| k.to_sym}
+    value = current_data.dig(*sym_keys)
+    logger.debug("Retrieved test data: #{keys.join('/')} => #{value.inspect}")
+    value
+  end
+
+  # Public: Checks if a data attribute is present.
+  #
+  # key - Symbol data attribute to check.
+  #
+  # Returns Boolean true or false.
+  def key?(key)
+    ret = current_data.key?(key.to_sym)
+    logger.debug("Key #{ret ? '' : 'not '}found in test data: #{key}")
+    ret
+  end
+
+  # Public: Gets a data attribute.
+  #
+  # key - Symbol data attribute to get.
+  #
+  # Returns Object value, or nil.
+  def [](key)
+    value = current_data[key.to_sym]
+    logger.debug("Retrieved test data: #{key} => #{value.inspect}")
+    value
+  end
+
+  # Public: Updates a data attribute.
+  #
+  # key   - Symbol data attribute to update.
+  # value - Object value to set.
+  #
+  # Returns Object value that was set.
+  def []=(key, value)
+    @test_case.lock(@w_lock_obj, 1.min) do
+      logger.debug("Assigning test data: #{key} => #{value.inspect}")
+      current_data[key.to_sym] = value
+      update(nil)
+      value
+    end
+  end
+
+  # Public: Fetches a data attribute.
+  #
+  # key  - Symbol data attribute to fetch.
+  # opts - Splat Array of additional options to `fetch'. Normally just a default to return if nil.
+  #
+  # Returns Object value, or returns the specified default if not exists.
+  # Raises an Exception if the value did not exist and no default specified.
+  def fetch(key, *opts)
+    begin
+      value = current_data.fetch(key.to_sym, *opts)
+      logger.debug("Fetched test data: #{key} => #{value.inspect}")
+    rescue IndexError
+      raise "Key not found in test data: #{key}"
+    end
+    value
+  end
+
+  # Public: Deletes a data attribute.
+  #
+  # key - Symbol data attribute to delete.
+  #
+  # Returns Object value of deleted attribute.
+  def delete(key)
+    @test_case.lock(@w_lock_obj, 1.min) do
+      value = current_data.delete(key.to_sym)
+      logger.debug("Deleted test data: #{key} => #{value.inspect}")
+      update(nil)
+      value
+    end
+  end
+
+  # Public: Updates test data.
+  #
+  # obj - Hash from which to update test data.
+  #
+  # Returns updated test data Hash.
+  def update(obj)
+    @test_case.lock(@w_lock_obj, 1.min) do
+      unless obj.nil?
+        logger.debug("Updating test data: #{obj.inspect}")
+        current_data.update(@test_case.rubify(obj, snake_case: false))
+      end
+      data = @test_case.jsonify(@data, camel_case: false)
+      resp = delay(RW_DELAY_MS) do
+        @test_case.send(:tmc_put, "/api/jobs/#{@job}/data", json: data)
+      end
+      begin
+        if @data_file.nil?
+          @test_case.refute_nil(resp)
+        elsif resp.nil?
+          # Fall back to local data file, if provided
+          path = File.join(Dir.pwd, 'conf', @data_file)
+          File.write(path, JSON.generate(data, {indent: '  ', space: ' ', object_nl: "\n", array_nl: "\n"}))
+        end
+      rescue => e
+        raise "Failed to update test data! #{e.message}\n\t#{e.backtrace.join("\n\t")}"
+      end
+      @data
+    end
+  end
+
+  # Public: Refreshes test data.
+  #
+  # Returns refreshed test data Hash.
+  def refresh
+    json = delay(RW_DELAY_MS) do
+      @test_case.send(:tmc_get, "/api/jobs/#{@job}/data?seq=#{@seq}")
+    end
+    no_update = false
+    begin
+      if @data_file.nil?
+        @test_case.refute_nil(json)
+        json = json['GetTestDataResult'] unless json['GetTestDataResult'].nil?  # server bug
+        no_update = json['testData'].nil?
+        unless no_update
+          @seq = json['seq'] unless json['seq'].nil?
+          json = json['testData']
+        end
+      elsif json.nil?
+        # Fall back to local data file, if provided
+        path = File.join(Dir.pwd, 'conf', @data_file)
+        json = JSON.parse(File.read(path))
+      end
+      unless no_update
+        @data.clear
+        @data.update(@test_case.rubify(json, snake_case: false))
+      end
+    rescue => e
+      raise "Failed to refresh test data! #{e.message}\n\t#{e.backtrace.join("\n\t")}"
+    end
+    @data
+  end
+
+  private
+
+  # Internal: Delays until the given time in ms has elapsed since the last delay, then executes the specified block.
+  #
+  # ms - Integer ms delay to enforce.
+  #
+  # Returns the Object result of the given block.
+  def delay(ms, &block)
+    delay_ms = (ms / 1000.0) - (Time.now - (@last_rw_time || Time.at(0)))
+    if delay_ms > 0
+      sleep(delay_ms)
+    end
+    ret = block.call
+    @last_rw_time = Time.now
+    ret
+  end
+
+  # Internal: Provides access to test case logger.
+  def logger
+    @test_case.logger
+  end
+
+  # Internal: Gets current test data.
+  def current_data
+    refresh if @promiscuous || @seq > 0  # TODO: always refresh
+    @data
+  end
+
+end