qa_server 7.2.0 → 7.5.1

data/app/services/qa_server/performance_graph_data_service.rb

@@ -73,7 +73,7 @@ module QaServer
       # @param authority_name [String] name of an authority
       # @param action [Symbol] :search, :fetch, or :all_actions
       # @param averages [Hash] existing data for each hour
-      # @returns [Hash] existing hourly data with the last hour updated
+      # @returns [Hash] existing day data with the last hour updated
       # @example returns
       #   { 0: { hour: '1400', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
       #     1: { hour: '1500', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
@@ -88,39 +88,39 @@ module QaServer
         averages
       end
 
-      private
+    private
 
-        def records_by(authority_name, action, time_period)
-          where_clause = { dt_stamp: time_period }
-          where_clause[:authority] = authority_name unless authority_name.nil? || authority_name == ALL_AUTH
-          where_clause[:action] = action unless action.nil? || action == ALL_ACTIONS
-          performance_data_class.where(where_clause)
-        end
+      def records_by(authority_name, action, time_period)
+        where_clause = { dt_stamp: time_period }
+        where_clause[:authority] = authority_name unless authority_name.nil? || authority_name == ALL_AUTH
+        where_clause[:action] = action unless action.nil? || action == ALL_ACTIONS
+        performance_data_class.where(where_clause)
+      end
 
-        def performance_by_hour_label(idx, start_hour)
-          if idx == 23
-            I18n.t('qa_server.monitor_status.performance.now')
-          elsif ((idx + 1) % 2).zero?
-            (start_hour.hour * 100).to_s
-          else
-            ""
-          end
+      def performance_by_hour_label(idx, start_hour)
+        if idx == 23
+          I18n.t('qa_server.monitor_status.performance.now')
+        elsif ((idx + 1) % 2).zero?
+          (start_hour.hour * 100).to_s
+        else
+          " "
         end
+      end
 
-        def performance_by_day_label(idx, start_day)
-          if idx == 29
-            I18n.t('qa_server.monitor_status.performance.today')
-          elsif ((idx + 1) % 5).zero?
-            start_day.strftime("%m-%d")
-          else
-            ""
-          end
+      def performance_by_day_label(idx, start_day)
+        if idx == 29
+          I18n.t('qa_server.monitor_status.performance.today')
+        elsif ((idx + 1) % 5).zero?
+          start_day.strftime("%m-%d")
+        else
+          " "
         end
+      end
 
-        def calculate_from_records(records, range_idx, range_label)
-          stats = stats_calculator_class.new(records).calculate_average_stats
-          { STATS => stats, range_idx => range_label }
-        end
+      def calculate_from_records(records, range_idx, range_label)
+        stats = stats_calculator_class.new(records).calculate_average_stats
+        { STATS => stats, range_idx => range_label }
+      end
     end
   end
 end

data/app/services/qa_server/performance_graphing_service.rb

@@ -31,7 +31,7 @@ module QaServer
       # @param action [Symbol] action performed by the request (e.g. :search, :fetch, :all_actions)
       # @param data [Hash] data to use to generate the graph
       # @see QaServer::PerformanceGraphDataService.calculate_last_12_months
-      def generate_monthly_graph(authority_name: ALL_AUTH, action:, data:)
+      def generate_year_graph(authority_name: ALL_AUTH, action:, data:)
         gruff_data = rework_performance_data_for_gruff(data, BY_MONTH)
         create_gruff_graph(gruff_data,
                            performance_graph_full_path(authority_name, action, FOR_YEAR),
@@ -44,7 +44,7 @@ module QaServer
       # @param action [Symbol] action performed by the request (e.g. :search, :fetch, :all_actions)
       # @param data [Hash] data to use to generate the graph
       # @see QaServer::PerformanceGraphDataService.calculate_last_30_days
-      def generate_daily_graph(authority_name: ALL_AUTH, action:, data:)
+      def generate_month_graph(authority_name: ALL_AUTH, action:, data:)
         gruff_data = rework_performance_data_for_gruff(data, BY_DAY)
         create_gruff_graph(gruff_data,
                            performance_graph_full_path(authority_name, action, FOR_MONTH),
@@ -56,76 +56,76 @@ module QaServer
       # @param action [Symbol] action performed by the request (e.g. :search, :fetch, :all_actions)
       # @param data [Hash] data to use to generate the graph
       # @see QaServer::PerformanceGraphDataService.calculate_last_24_hours
-      def generate_hourly_graph(authority_name: ALL_AUTH, action:, data:)
+      def generate_day_graph(authority_name: ALL_AUTH, action:, data:)
         gruff_data = rework_performance_data_for_gruff(data, BY_HOUR)
         create_gruff_graph(gruff_data,
                            performance_graph_full_path(authority_name, action, FOR_DAY),
                            I18n.t('qa_server.monitor_status.performance.x_axis_hour'))
       end
 
-      private
+    private
 
-        def performance_graph_full_path(authority_name, action, time_period)
-          graph_full_path(graph_filename(authority_name, action, time_period))
-        end
+      def performance_graph_full_path(authority_name, action, time_period)
+        graph_full_path(graph_filename(authority_name, action, time_period))
+      end
 
-        def graph_filename(authority_name, action, time_period)
-          "performance_of_#{authority_name}_#{action}_for_#{time_period}_graph.png"
-        end
+      def graph_filename(authority_name, action, time_period)
+        "performance_of_#{authority_name}_#{action}_for_#{time_period}_graph.png"
+      end
 
-        def rework_performance_data_for_gruff(performance_data, label_key)
-          labels = {}
-          # full_load_data = []
-          retrieve_data = []
-          graph_load_data = []
-          normalization_data = []
-          performance_data.each do |i, data|
-            labels[i] = data[label_key]
-            retrieve_data << data[STATS][AVG_RETR]
-            graph_load_data << graph_load_time(data)
-            normalization_data << data[STATS][AVG_NORM]
-          end
-          [labels, retrieve_data, graph_load_data, normalization_data]
+      def rework_performance_data_for_gruff(performance_data, label_key)
+        labels = {}
+        # full_load_data = []
+        retrieve_data = []
+        graph_load_data = []
+        normalization_data = []
+        performance_data.each do |i, data|
+          labels[i] = data[label_key]
+          retrieve_data << data[STATS][AVG_RETR]
+          graph_load_data << graph_load_time(data)
+          normalization_data << data[STATS][AVG_NORM]
         end
+        [labels, retrieve_data, graph_load_data, normalization_data]
+      end
 
-        def graph_load_time(data)
-          # For some sense of backward compatibility and to avoid losing the usefulness of previously collected data,
-          # create the graph using the old :load stat when both :retrieve and :graph_load are 0. If the value truly
-          # is 0, then :load will also be 0.
-          # NOTE: It's ok to use AVG_RETR for the retrieve data point because it is 0.
-          # rubocop:disable Style/TernaryParentheses
-          (data[STATS][AVG_RETR].zero? && data[STATS][AVG_GRPH].zero?) ? data[STATS][AVG_LOAD] : data[STATS][AVG_GRPH]
-          # rubocop:enable Style/TernaryParentheses
-        end
+      def graph_load_time(data)
+        # For some sense of backward compatibility and to avoid losing the usefulness of previously collected data,
+        # create the graph using the old :load stat when both :retrieve and :graph_load are 0. If the value truly
+        # is 0, then :load will also be 0.
+        # NOTE: It's ok to use AVG_RETR for the retrieve data point because it is 0.
+        # rubocop:disable Style/TernaryParentheses
+        (data[STATS][AVG_RETR].zero? && data[STATS][AVG_GRPH].zero?) ? data[STATS][AVG_LOAD] : data[STATS][AVG_GRPH]
+        # rubocop:enable Style/TernaryParentheses
+      end
 
-        def performance_graph_theme(g, x_axis_label)
-          g.theme_pastel
-          g.colors = [QaServer.config.performance_normalization_color,
-                      QaServer.config.performance_graph_load_color,
-                      QaServer.config.performance_retrieve_color]
-          g.marker_font_size = 12
-          g.x_axis_increment = 10
-          g.x_axis_label = x_axis_label
-          g.y_axis_label = I18n.t('qa_server.monitor_status.performance.y_axis_ms')
-          g.minimum_value = 0
-          g.maximum_value = QaServer.config.performance_y_axis_max
-        end
+      def performance_graph_theme(g, x_axis_label)
+        g.theme_pastel
+        g.colors = [QaServer.config.performance_normalization_color,
+                    QaServer.config.performance_graph_load_color,
+                    QaServer.config.performance_retrieve_color]
+        g.marker_font_size = 12
+        g.x_axis_increment = 10
+        g.x_axis_label = x_axis_label
+        g.y_axis_label = I18n.t('qa_server.monitor_status.performance.y_axis_ms')
+        g.minimum_value = 0
+        g.maximum_value = QaServer.config.performance_y_axis_max
+      end
 
-        def create_gruff_graph(performance_data, performance_graph_full_path, x_axis_label)
-          g = Gruff::StackedBar.new
-          performance_graph_theme(g, x_axis_label)
-          g.labels = performance_data[0]
-          g.data(I18n.t('qa_server.monitor_status.performance.retrieve_time_ms'), performance_data[1])
-          g.data(I18n.t('qa_server.monitor_status.performance.graph_load_time_ms'), performance_data[2])
-          g.data(I18n.t('qa_server.monitor_status.performance.normalization_time_ms'), performance_data[3])
-          g.write performance_graph_full_path
-        end
+      def create_gruff_graph(performance_data, performance_graph_full_path, x_axis_label)
+        g = Gruff::StackedBar.new
+        performance_graph_theme(g, x_axis_label)
+        g.labels = performance_data[0]
+        g.data(I18n.t('qa_server.monitor_status.performance.retrieve_time_ms'), performance_data[1])
+        g.data(I18n.t('qa_server.monitor_status.performance.graph_load_time_ms'), performance_data[2])
+        g.data(I18n.t('qa_server.monitor_status.performance.normalization_time_ms'), performance_data[3])
+        g.write performance_graph_full_path
+      end
 
-        def log_failure(authority_name, action, time_period)
-          relative_path = performance_graph_image_path(authority_name: authority_name, action: action, time_period: time_period)
-          exists = performance_graph_image_exists?(authority_name: authority_name, action: action, time_period: time_period)
-          QaServer.config.monitor_logger.warn("FAILED to write performance graph at #{relative_path}") unless exists
-        end
+      def log_failure(authority_name, action, time_period)
+        relative_path = performance_graph_image_path(authority_name: authority_name, action: action, time_period: time_period)
+        exists = performance_graph_image_exists?(authority_name: authority_name, action: action, time_period: time_period)
+        QaServer.config.monitor_logger.warn("FAILED to write performance graph at #{relative_path}") unless exists
+      end
     end
   end
 end

data/app/services/qa_server/performance_per_byte_calculator_service.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+# This class calculates min, max, average stats for load, normalization, and full request times for a given set of performance records.
+require 'matrix'
+module QaServer
+  class PerformancePerByteCalculatorService
+    include QaServer::PerformanceHistoryDataKeys
+
+    TIME  = 0
+    BYTES = 1
+
+    attr_reader :n, :stats, :records
+
+    # @param records [Array <Qa::PerformanceHistory>] set of records used to calculate the statistics
+    def initialize(records:, n:)
+      @records = records
+      @n = n
+      @stats = {}
+    end
+
+    # Calculate performance statistics with percentiles.  Min is at the 10th percentile.  Max is at the 90th percentile.
+    # @return [Hash] hash of the statistics
+    # @example
+    #   { data_raw_bytes_from_source: [16271, 16271],
+    #     retrieve_bytes_per_ms: [67.24433786890475, 55.51210410757532],
+    #     retrieve_ms_per_byte: [0.014871140555351083, 0.018014089288745542]
+    #     graph_load_bytes_per_ms_ms: [86.74089418722461, 54.97464153778724],
+    #     graph_load_ms_per_byte: [0.011528587632974647, 0.018190205011389522],
+    #     normalization_bytes_per_ms: [64.70169466560836, 89.25337465693322],
+    #     normalization_ms_per_byte: [0.01530700843338457, 0.015455545718983178]
+    #   }
+    def calculate
+      extract_bytes
+      calculate_retrieve_stats
+      calculate_graph_load_stats
+      calculate_normalization_stats
+      stats
+    end
+
+  private
+
+    def extract_bytes
+      stats[SRC_BYTES] = retrieve_data.count.zero? ? 0 : retrieve_data.map { |d| d[BYTES] }
+    end
+
+    def calculate_retrieve_stats
+      stats[BPMS_RETR] = calculate_bytes_per_ms(retrieve_data)
+      stats[MSPB_RETR] = calculate_ms_per_byte(retrieve_data)
+    end
+
+    def calculate_graph_load_stats
+      stats[BPMS_GRPH] = calculate_bytes_per_ms(graph_load_data)
+      stats[MSPB_GRPH] = calculate_ms_per_byte(graph_load_data)
+    end
+
+    def calculate_normalization_stats
+      stats[BPMS_NORM] = calculate_bytes_per_ms(norm_data)
+      stats[MSPB_NORM] = calculate_ms_per_byte(norm_data)
+    end
+
+    def calculate_bytes_per_ms(data)
+      return 0 if data.count.zero?
+      return data[0][BYTES] / d[0][TIME] if data.count == 1
+      data.map { |d| d[BYTES] / d[TIME] }
+    end
+
+    def calculate_ms_per_byte(data)
+      return 0 if data.count.zero?
+      return data[0][TIME] / d[0][BYTES] if data.count == 1
+      data.map { |d| d[TIME] / d[BYTES] }
+    end
+
+    def data(column)
+      records.where.not(column => nil).order(dt_stamp: :desc).limit(n).pluck(column, :size_bytes)
+    end
+
+    def retrieve_data
+      @retrieve_data ||= data(:retrieve_time_ms)
+    end
+
+    def graph_load_data
+      @graph_data ||= data(:graph_load_time_ms)
+    end
+
+    def norm_data
+      @norm_data ||= data(:normalization_time_ms)
+    end
+  end
+end

data/app/services/qa_server/performance_per_byte_data_service.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+# This class calculates performance stats based on size of data.
+module QaServer
+  class PerformancePerByteDataService
+    class << self
+      include QaServer::PerformanceHistoryDataKeys
+
+      class_attribute :stats_calculator_class, :performance_data_class
+      self.stats_calculator_class = QaServer::PerformancePerByteCalculatorService
+      self.performance_data_class = QaServer::PerformanceHistory
+
+      # Performance data based on size of data.
+      # @param authority_name [String] name of an authority
+      # @param action [Symbol] :search, :fetch, or :all_actions
+      # @param n [Integer] calculate stats for last n records
+      # @returns [Hash] performance statistics based on size of data
+      # @example returns for n=2
+      #   { data_raw_bytes_from_source: [16271, 16271],
+      #     retrieve_bytes_per_ms: [67.24433786890475, 55.51210410757532],
+      #     retrieve_ms_per_byte: [0.014871140555351083, 0.018014089288745542]
+      #     graph_load_bytes_per_ms_ms: [86.74089418722461, 54.97464153778724],
+      #     graph_load_ms_per_byte: [0.011528587632974647, 0.018190205011389522],
+      #     normalization_bytes_per_ms: [64.70169466560836, 89.25337465693322],
+      #     normalization_ms_per_byte: [0.01530700843338457, 0.015455545718983178]
+      #   }
+      def calculate(authority_name:, action:, n: 10)
+        records = records_by(authority_name, action)
+        stats_calculator_class.new(records: records, n: n).calculate
+      end
+
+    private
+
+      def records_by(authority_name, action)
+        where_clause = {}
+        where_clause[:authority] = authority_name unless authority_name.nil? || authority_name == ALL_AUTH
+        where_clause[:action] = action unless action.nil? || action == ALL_ACTIONS
+        performance_data_class.where(where_clause)
+      end
+    end
+  end
+end

data/app/services/qa_server/scenarios_loader_service.rb

@@ -32,7 +32,7 @@ module QaServer
 
     def self.load_config(authority_name, status_log)
       scenarios_config = YAML.load_file(scenario_path(authority_name))
-      unless scenarios_config.present?
+      if scenarios_config.blank?
         status_log.add(authority_name: authority_name,
                        status: QaServer::ScenarioValidator::FAIL,
                        error_message: "Unable to load scenarios for authority '#{authority_name}'; cause: UNKNOWN")

data/app/services/qa_server/time_period_service.rb

@@ -63,31 +63,31 @@ module QaServer
         where_with_authority(where_clause, auth_name, auth_table)
       end
 
-      private
+    private
 
-        def where_for_dt_stamp(dt_table, dt_column, time_period)
-          end_range = QaServer::TimeService.current_time
-          start_range = end_range - time_period
-          where_clause = { dt_column => start_range..end_range }
-          where_clause = { dt_table => where_clause } unless dt_table.nil?
-          where_clause
-        end
+      def where_for_dt_stamp(dt_table, dt_column, time_period)
+        end_range = QaServer::TimeService.current_time
+        start_range = end_range - time_period
+        where_clause = { dt_column => start_range..end_range }
+        where_clause = { dt_table => where_clause } unless dt_table.nil?
+        where_clause
+      end
 
-        def where_with_authority(where_clause, auth_name, auth_table)
-          return where_clause if auth_name.nil?
-          if auth_table.nil?
-            where_clause[:authority] = auth_name
-          else
-            where_clause[auth_table] = { authority: auth_name }
-          end
-          where_clause
+      def where_with_authority(where_clause, auth_name, auth_table)
+        return where_clause if auth_name.nil?
+        if auth_table.nil?
+          where_clause[:authority] = auth_name
+        else
+          where_clause[auth_table] = { authority: auth_name }
         end
+        where_clause
+      end
 
-        def validate_params(auth_name, auth_table, dt_table)
-          raise ArgumentError, "Do not specify auth_table when auth_name is not specified" if auth_table.present? && auth_name.nil?
-          return if auth_name.nil?
-          raise ArgumentError, "Either both table names need to be specified or neither" if auth_table.present? ^ dt_table.present?
-        end
+      def validate_params(auth_name, auth_table, dt_table)
+        raise ArgumentError, "Do not specify auth_table when auth_name is not specified" if auth_table.present? && auth_name.nil?
+        return if auth_name.nil?
+        raise ArgumentError, "Either both table names need to be specified or neither" if auth_table.present? ^ dt_table.present?
+      end
     end
   end
 end

data/app/validators/qa_server/scenario_validator.rb

@@ -8,6 +8,7 @@ module QaServer
 
     VALIDATE_CONNECTION = :connection
     VALIDATE_ACCURACY = :accuracy
+    VALIDATE_ACCURACY_COMPARISON = :accuracy_comparison
     ALL_VALIDATIONS = :all_validations
     DEFAULT_VALIDATION_TYPE = VALIDATE_CONNECTION
 
@@ -33,92 +34,103 @@ module QaServer
       log
     end
 
-    private
-
-      # Log the structure of the scenario and status of a test run.
-      # @param [Hash] status_info holding information to be logged
-      # @see QaServer::ScenarioLogger
-      def log(status_info = {})
-        status_info[:authority_name] = authority_name
-        status_info[:validation_type] = scenario_validation_type
-        status_info[:subauth] = subauthority_name
-        status_info[:service] = service
-        status_info[:action] = action
-        status_info[:url] = url
-        status_log.add(status_info)
-      end
-
-      def run_accuracy_scenario
-        # ABSTRACT - must be implemented by scenario validator for specific types
-      end
-
-      def run_connection_scenario
-        # ABSTRACT - must be implemented by scenario validator for specific types
-      end
-
-      # Runs the test in the block passed by the specific scenario type.
-      # @return [Symbol] :good (PASS) or :unknown (UNKNOWN) based on whether enough results were returned
-      def test_connection(min_expected_size: MIN_EXPECTED_SIZE, scenario_type_name:)
-        dt_start = QaServer::TimeService.current_time
-        results = yield if block_given?
-        dt_end = QaServer::TimeService.current_time
-        actual_size = results.to_s.length
-        status = actual_size > min_expected_size ? PASS : UNKNOWN
-        errmsg = status == PASS ? '' : "#{scenario_type_name.capitalize} scenario unknown status; cause: Results actual size (#{actual_size} < expected size (#{min_expected_size})"
-        log(status: status, error_message: errmsg, normalization_run_time: (dt_end - dt_start)) # TODO: need to get run times from results
-      rescue Exception => e
-        dt_end = QaServer::TimeService.current_time
-        log(status: FAIL, error_message: "Exception executing #{scenario_type_name} scenario; cause: #{e.message}", request_run_time: (dt_end - dt_start))
-      end
-
-      def authority
-        scenario.authority
-      end
-
-      def authority_name
-        scenario.authority_name.upcase
-      end
-
-      def subauthority_name
-        scenario.subauthority_name
-      end
-
-      def service
-        scenario.service
-      end
-
-      def url
-        scenario.url
-      end
-
-      def action
-        # ABSTRACT - must be implemented by scenario validator for specific types
-      end
-
-      def scenario_validation_type
-        return VALIDATE_CONNECTION if connection_scenario?
-        return VALIDATE_ACCURACY if accuracy_scenario?
-        nil
-      end
-
-      def validating_connections?
-        return true if validation_type == VALIDATE_CONNECTION || validation_type == ALL_VALIDATIONS
-        false
-      end
-
-      def validating_accuracy?
-        return true if validation_type == VALIDATE_ACCURACY || validation_type == ALL_VALIDATIONS
-        false
-      end
-
-      def accuracy_scenario?
-        # ABSTRACT define in specific scenario type validator (i.e. QaServer::TermScenarioValidator, QaServer::SearchScenarioValidator)
-        false
-      end
-
-      def connection_scenario?
-        # ABSTRACT define in specific scenario type validator (i.e. QaServer::TermScenarioValidator, QaServer::SearchScenarioValidator)
-        false
-      end
+  private
+
+    # Log the structure of the scenario and status of a test run.
+    # @param [Hash] status_info holding information to be logged
+    # @see QaServer::ScenarioLogger
+    def log(status_info = {})
+      status_info[:authority_name] = authority_name
+      status_info[:validation_type] = scenario_validation_type
+      status_info[:subauth] = subauthority_name
+      status_info[:service] = service
+      status_info[:action] = action
+      status_info[:url] = url
+      status_info[:request_data] = request_data
+      status_log.add(status_info)
+    end
+
+    def run_accuracy_scenario
+      # ABSTRACT - must be implemented by scenario validator for specific types
+    end
+
+    def run_connection_scenario
+      # ABSTRACT - must be implemented by scenario validator for specific types
+    end
+
+    def request_data
+      # ABSTRACT - must be implemented by scenario validator for specific types
+    end
+
+    # Runs the test in the block passed by the specific scenario type.
+    # @return [Symbol] :good (PASS) or :unknown (UNKNOWN) based on whether enough results were returned
+    def test_connection(min_expected_size: MIN_EXPECTED_SIZE, scenario_type_name:)
+      dt_start = QaServer::TimeService.current_time
+      results = yield if block_given?
+      dt_end = QaServer::TimeService.current_time
+      actual_size = results.to_s.length
+      status = actual_size > min_expected_size ? PASS : UNKNOWN
+      errmsg = status == PASS ? '' : "#{scenario_type_name.capitalize} scenario unknown status; cause: Results actual size (#{actual_size} < expected size (#{min_expected_size})"
+      log(status: status, error_message: errmsg, normalization_run_time: (dt_end - dt_start)) # TODO: need to get run times from results
+    rescue Exception => e
+      dt_end = QaServer::TimeService.current_time
+      log(status: FAIL, error_message: "Exception executing #{scenario_type_name} scenario; cause: #{e.message}", request_run_time: (dt_end - dt_start))
+    end
+
+    # @return [Qa::Authorities::LinkedData::GenericAuthority] authority instance the scenarios run against
+    def authority
+      scenario.authority
+    end
+
+    # @return [Symbol] name of the authority (e.g. :CERL_LD4L_CACHE)
+    def authority_name
+      scenario.authority_name.upcase
+    end
+
+    # @return [String] name of the subauthority (e.g. "person")
+    def subauthority_name
+      scenario.subauthority_name
+    end
+
+    # @return [String] service providing the data (e.g. "ld4l_cache" or "direct")
+    def service
+      scenario.service
+    end
+
+    # @return [String] relative URL with ready to execute (e.g. ""/authorities/search/linked_data/cerl_ld4l_cache/person?q=office")
+    def url
+      scenario.url
+    end
+
+    # @return [String] action being tested (e.g. "search" or "term")
+    def action
+      # ABSTRACT - must be implemented by scenario validator for specific types
+    end
+
+    def scenario_validation_type
+      return VALIDATE_CONNECTION if connection_scenario?
+      return VALIDATE_ACCURACY if accuracy_scenario?
+      nil
+    end
+
+    def validating_connections?
+      return true if validation_type == VALIDATE_CONNECTION || validation_type == ALL_VALIDATIONS
+      false
+    end
+
+    def validating_accuracy?
+      return true if [VALIDATE_ACCURACY, VALIDATE_ACCURACY_COMPARISON, ALL_VALIDATIONS].include? validation_type
+      false
+    end
+
+    def accuracy_scenario?
+      # ABSTRACT define in specific scenario type validator (i.e. QaServer::TermScenarioValidator, QaServer::SearchScenarioValidator)
+      false
+    end
+
+    def connection_scenario?
+      # ABSTRACT define in specific scenario type validator (i.e. QaServer::TermScenarioValidator, QaServer::SearchScenarioValidator)
+      false
+    end
   end
 end