qa_server 7.2.1 → 7.6.0

data/app/jobs/qa_server/history_graph_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+# Job to generate the graph of historical test runs per authority.
+module QaServer
+  class HistoryGraphJob < ApplicationJob
+    queue_as :default
+
+    class_attribute :graphing_service
+    self.graphing_service = QaServer::HistoryGraphingService
+
+    def perform(data:)
+      # checking active_job_id? prevents race conditions for long running jobs
+      generate_graph(data) if QaServer::JobIdCache.active_job_id?(job_key: job_key, job_id: job_id)
+    end
+
+  private
+
+    def generate_graph(data)
+      QaServer.config.monitor_logger.debug("(#{self.class}##{__method__}-#{job_id}) - GENERATING historical summary graph")
+      graphing_service.generate_graph(data)
+      QaServer.config.monitor_logger.debug("(#{self.class}##{__method__}-#{job_id}) COMPLETED historical summary graph generation")
+      QaServer::JobIdCache.reset_job_id(job_key: job_key)
+    end
+
+    def job_key
+      "QaServer::HistoryGraphJob--job_id"
+    end
+  end
+end

data/app/jobs/qa_server/monitor_tests_job.rb

@@ -10,49 +10,29 @@ module QaServer
     self.scenario_run_registry_class = QaServer::ScenarioRunRegistry
 
     def perform
-      Rails.cache.fetch("QaServer::MonitorStatusController/latest_test_run_from_cache", expires_in: QaServer::CacheExpiryService.cache_expiry, race_condition_ttl: 5.minutes, force: true) do
-        job_id = SecureRandom.uuid
-        monitor_tests_job_id = job_id unless monitor_tests_job_id
-        run_tests if monitor_tests_job_id == job_id # avoid race conditions
-        scenario_run_registry_class.latest_run
-      end
+      # checking active_job_id? prevents race conditions for long running jobs
+      run_tests if QaServer::JobIdCache.active_job_id?(job_key: job_key, job_id: job_id, expires_in: 2.hours)
     end
 
-    private
+  private
 
-      def run_tests
-        QaServer.config.monitor_logger.debug("(#{self.class}##{__method__}-#{job_id}) RUNNING monitoring tests")
-        validate(authorities_list)
-        log_results(authorities_list, status_log.to_a)
-        scenario_run_registry_class.save_run(scenarios_results: status_log.to_a)
-        QaServer.config.monitor_logger.debug("(#{self.class}##{__method__}-#{job_id}) COMPLETED monitoring tests")
-        QaServer.config.performance_cache.write_all # write out cache after completing tests
-        reset_monitor_tests_job_id
-      end
-
-      # @return [String, Boolean] Returns job id of the job currently running tests; otherwise, false if tests are not running
-      def monitor_tests_job_id
-        Rails.cache.fetch("QaServer:monitor_tests-job_id", expires_in: 2.hours, race_condition_ttl: 5.minutes) { false }
-      end
-
-      # Set the id of the job that will run the tests.
-      # @param job_id [String] UUID for job running the tests
-      def monitor_tests_job_id=(job_id)
-        # check to see if there is a current job already running tests
-        current_job_id = Rails.cache.fetch("QaServer:monitor_tests-job_id", expires_in: 2.hours, race_condition_ttl: 5.seconds) { job_id }
-
-        # current_job_id may be false meaning tests are not currently running; in which case, it is ok to force set job_id
-        Rails.cache.fetch("QaServer:monitor_tests-job_id", expires_in: 2.hours, race_condition_ttl: 30.seconds, force: true) { job_id } unless current_job_id
-      end
+    def run_tests
+      QaServer.config.monitor_logger.debug("(#{self.class}##{__method__}-#{job_id}) RUNNING monitoring tests")
+      validate(authorities_list)
+      log_results(authorities_list, status_log.to_a)
+      scenario_run_registry_class.save_run(scenarios_results: status_log.to_a)
+      QaServer.config.monitor_logger.debug("(#{self.class}##{__method__}-#{job_id}) COMPLETED monitoring tests")
+      QaServer.config.performance_cache.write_all # write out cache after completing tests
+      QaServer::JobIdCache.reset_job_id(job_key: job_key)
+    end
 
-      # Set job id for monitor tests to false indicating that tests are not currently running
-      def reset_monitor_tests_job_id
-        Rails.cache.fetch("QaServer:monitor_tests-job_id", expires_in: 2.hours, race_condition_ttl: 30.seconds, force: true) { false }
-      end
+    def log_results(authorities_list, results)
+      QaServer.config.monitor_logger.warn("(#{self.class}##{__method__}-#{job_id}) authorities_list is empty") if authorities_list&.empty?
+      QaServer.config.monitor_logger.warn("(#{self.class}##{__method__}-#{job_id}) test results are empty") if results&.empty?
+    end
 
-      def log_results(authorities_list, results)
-        QaServer.config.monitor_logger.warn("(#{self.class}##{__method__}-#{job_id}) authorities_list is empty") if authorities_list&.empty?
-        QaServer.config.monitor_logger.warn("(#{self.class}##{__method__}-#{job_id}) test results are empty") if results&.empty?
-      end
+    def job_key
+      "QaServer::MonitorTestsJob--job_id"
+    end
   end
 end

data/app/jobs/qa_server/performance_day_graph_job.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+# Job to generate the performance day graph covering the last 24 hours.
+module QaServer
+  class PerformanceDayGraphJob < ApplicationJob
+    include QaServer::PerformanceHistoryDataKeys
+
+    queue_as :default
+
+    class_attribute :authority_list_class, :graph_data_service, :graphing_service
+    self.authority_list_class = QaServer::AuthorityListerService
+    self.graph_data_service = QaServer::PerformanceGraphDataService
+    self.graphing_service = QaServer::PerformanceGraphingService
+
+    def perform
+      # checking active_job_id? prevents race conditions for long running jobs
+      generate_graphs_for_authorities if QaServer::JobIdCache.active_job_id?(job_key: job_key, job_id: job_id)
+    end
+
+  private
+
+    def generate_graphs_for_authorities
+      QaServer.config.monitor_logger.debug("(#{self.class}-#{job_id}) - GENERATING performance day graph")
+      auths = authority_list_class.authorities_list
+      generate_graphs_for_authority(authority_name: ALL_AUTH) # generates graph for all authorities
+      auths.each { |authname| generate_graphs_for_authority(authority_name: authname) }
+      QaServer.config.monitor_logger.debug("(#{self.class}-#{job_id}) COMPLETED performance day graph generation")
+      QaServer::JobIdCache.reset_job_id(job_key: job_key)
+    end
+
+    def generate_graphs_for_authority(authority_name:)
+      [SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
+        hash[action] = generate_24_hour_graph(authority_name: authority_name, action: action)
+      end
+    end
+
+    def generate_24_hour_graph(authority_name:, action:)
+      data = graph_data_service.calculate_last_24_hours(authority_name: authority_name, action: action)
+      graphing_service.generate_day_graph(authority_name: authority_name, action: action, data: data)
+    end
+
+    def job_key
+      "QaServer::PerformanceDayGraphJob--job_id"
+    end
+  end
+end

data/app/jobs/qa_server/performance_month_graph_job.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+# Job to generate the performance month graph covering the last 30 days.
+module QaServer
+  class PerformanceMonthGraphJob < ApplicationJob
+    include QaServer::PerformanceHistoryDataKeys
+
+    queue_as :default
+
+    class_attribute :authority_list_class, :graph_data_service, :graphing_service
+    self.authority_list_class = QaServer::AuthorityListerService
+    self.graph_data_service = QaServer::PerformanceGraphDataService
+    self.graphing_service = QaServer::PerformanceGraphingService
+
+    def perform
+      # checking active_job_id? prevents race conditions for long running jobs
+      generate_graphs_for_authorities if QaServer::JobIdCache.active_job_id?(job_key: job_key, job_id: job_id)
+    end
+
+  private
+
+    def generate_graphs_for_authorities
+      QaServer.config.monitor_logger.debug("(#{self.class}-#{job_id}) - GENERATING performance month graph")
+      auths = authority_list_class.authorities_list
+      generate_graphs_for_authority(authority_name: ALL_AUTH) # generates graph for all authorities
+      auths.each { |authname| generate_graphs_for_authority(authority_name: authname) }
+      QaServer.config.monitor_logger.debug("(#{self.class}-#{job_id}) COMPLETED performance month graph generation")
+      QaServer::JobIdCache.reset_job_id(job_key: job_key)
+    end
+
+    def generate_graphs_for_authority(authority_name:)
+      [SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
+        hash[action] = generate_30_day_graph(authority_name: authority_name, action: action)
+      end
+    end
+
+    def generate_30_day_graph(authority_name:, action:)
+      data = graph_data_service.calculate_last_30_days(authority_name: authority_name, action: action)
+      graphing_service.generate_month_graph(authority_name: authority_name, action: action, data: data)
+    end
+
+    def job_key
+      "QaServer::PerformanceMonthGraphJob--job_id"
+    end
+  end
+end

data/app/jobs/qa_server/performance_per_byte_job.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+# Job to generate the performance day graph covering the last 24 hours.
+module QaServer
+  class PerformancePerByteJob < ApplicationJob
+    include QaServer::PerformanceHistoryDataKeys
+
+    queue_as :default
+
+    class_attribute :authority_list_class, :data_service
+    self.authority_list_class = QaServer::AuthorityListerService
+    self.data_service = QaServer::PerformancePerByteDataService
+    # self.graphing_service = QaServer::PerformanceGraphingService
+
+    def perform(n: 10, action: :search, authority_complexity_ratings: {})
+      # checking active_job_id? prevents race conditions for long running jobs
+      generate_data_for_authorities(n, action, authority_complexity_ratings) if QaServer::JobIdCache.active_job_id?(job_key: job_key, job_id: job_id)
+    end
+
+  private
+
+    def generate_data_for_authorities(n, action, authority_complexity_ratings)
+      QaServer.config.monitor_logger.debug("(#{self.class}-#{job_id}) - GENERATING performance by byte data")
+      auths = authority_list_class.authorities_list
+      data = if action.nil?
+               # generate_data_for_authority(ALL_AUTH, n) # generates data for all authorities
+               auths.each_with_object({}) { |authname, hash| hash[authname] = generate_data_for_authority(authname, n) }
+             else
+               auths.each_with_object({}) { |authname, hash| hash[authname] = { action => generate_data(authname, action, n) } }
+             end
+      QaServer.config.monitor_logger.debug("(#{self.class}-#{job_id}) COMPLETED performance by byte data generation")
+      QaServer::JobIdCache.reset_job_id(job_key: job_key)
+      convert_to_csv(data, authority_complexity_ratings)
+    end
+
+    def generate_data_for_authority(authority_name, n)
+      [SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
+        hash[action] = generate_data(authority_name, action, n)
+      end
+    end
+
+    def generate_data(authority_name, action, n)
+      data_service.calculate(authority_name: authority_name, action: action, n: n)
+      # graphing_service.generate_day_graph(authority_name: authority_name, action: action, data: data)
+    end
+
+    # @param data [Hash] performance statistics based on size of data
+    # @param authority_complexity_ratings [Hash] complexity rating of the extended context included with results
+    # @example data
+    #   { data_raw_bytes_from_source: [16271, 16271],
+    #     retrieve_bytes_per_ms: [67.24433786890475, 55.51210410757532],
+    #     retrieve_ms_per_byte: [0.014871140555351083, 0.018014089288745542]
+    #     graph_load_bytes_per_ms_ms: [86.74089418722461, 54.97464153778724],
+    #     graph_load_ms_per_byte: [0.011528587632974647, 0.018190205011389522],
+    #     normalization_bytes_per_ms: [64.70169466560836, 89.25337465693322],
+    #     normalization_ms_per_byte: [0.01530700843338457, 0.015455545718983178]
+    #   }
+    def convert_to_csv(data, authority_complexity_ratings) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      performance_file = File.new('log/performance.csv', 'w')
+      performance_file.write("authority name, complexity_rating, action, size_bytes, ")
+      performance_file.write("retrieve_bytes_per_ms, graph_load_bytes_per_ms, normalize_bytes_per_ms, ")
+      performance_file.puts("retrieve_ms_per_byte, graph_load_ms_per_byte, normalize_ms_per_byte")
+      data.each do |auth_name, auth_data|
+        complexity_rating = authority_complexity_ratings.key?(auth_name) ? authority_complexity_ratings[auth_name] : "UNKNOWN"
+        auth_data.each do |action, action_data|
+          auth_action = "#{auth_name}, #{complexity_rating}, #{action}"
+          0.upto(action_data[:retrieve_bytes_per_ms].size - 1) do |idx|
+            performance_file.write(auth_action)
+            performance_file.write(", #{action_data[SRC_BYTES][idx]}")
+            performance_file.write(", #{action_data[BPMS_RETR][idx]}")
+            performance_file.write(", #{action_data[BPMS_GRPH][idx]}")
+            performance_file.write(", #{action_data[BPMS_NORM][idx]}")
+            performance_file.write(", #{action_data[MSPB_RETR][idx]}")
+            performance_file.write(", #{action_data[MSPB_GRPH][idx]}")
+            performance_file.puts(", #{action_data[MSPB_NORM][idx]}")
+          end
+        end
+      end
+      performance_file.close
+    end
+
+    def job_key
+      "QaServer::PerformanceByByteJob--job_id"
+    end
+  end
+end

data/app/jobs/qa_server/performance_year_graph_job.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+# Job to generate the performance year graph covering the last 12 months.
+module QaServer
+  class PerformanceYearGraphJob < ApplicationJob
+    include QaServer::PerformanceHistoryDataKeys
+
+    queue_as :default
+
+    class_attribute :authority_list_class, :graph_data_service, :graphing_service
+    self.authority_list_class = QaServer::AuthorityListerService
+    self.graph_data_service = QaServer::PerformanceGraphDataService
+    self.graphing_service = QaServer::PerformanceGraphingService
+
+    def perform
+      # checking active_job_id? prevents race conditions for long running jobs
+      generate_graphs_for_authorities if QaServer::JobIdCache.active_job_id?(job_key: job_key, job_id: job_id)
+    end
+
+  private
+
+    def generate_graphs_for_authorities
+      QaServer.config.monitor_logger.debug("(#{self.class}-#{job_id}) - GENERATING performance year graph")
+      auths = authority_list_class.authorities_list
+      generate_graphs_for_authority(authority_name: ALL_AUTH) # generates graph for all authorities
+      auths.each { |authname| generate_graphs_for_authority(authority_name: authname) }
+      QaServer.config.monitor_logger.debug("(#{self.class}-#{job_id}) COMPLETED performance year graph generation")
+      QaServer::JobIdCache.reset_job_id(job_key: job_key)
+    end
+
+    def generate_graphs_for_authority(authority_name:)
+      [SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
+        hash[action] = generate_12_month_graph(authority_name: authority_name, action: action)
+      end
+    end
+
+    def generate_12_month_graph(authority_name:, action:)
+      data = graph_data_service.calculate_last_12_months(authority_name: authority_name, action: action)
+      graphing_service.generate_year_graph(authority_name: authority_name, action: action, data: data)
+    end
+
+    def job_key
+      "QaServer::PerformanceYearGraphJob--job_id"
+    end
+  end
+end

data/app/loggers/qa_server/scenario_logger.rb

@@ -28,6 +28,7 @@ module QaServer
     # @option service [String] identifies the primary service provider (e.g. 'ld4l_cache', 'direct', etc.)
     # @option action [String] type of scenario (i.e. 'term', 'search')
     # @option url [String] example url that was used to test a specific term fetch or search query
+    # @option request_data [String] either the query for searches or the requested id/uri for fetch
     # @option error_message [String] error message if scenario failed
     # @option expected [Integer] the expected result (e.g. min size of result OR max position of subject within results)
     # @option actual [Integer] the actual result (e.g. actual size of results OR actual position of subject within results)
@@ -44,12 +45,14 @@ module QaServer
                 service: status_info[:service] || '',
                 action: status_info[:action] || '',
                 url: status_info[:url] || '',
+                request_data: status_info[:request_data] || '',
                 expected: status_info[:expected] || nil,
                 actual: status_info[:actual] || nil,
                 target: status_info[:target] || nil,
                 err_message: status_info[:error_message] || '',
                 request_run_time: status_info[:request_run_time] || nil,
-                normalization_run_time: status_info[:normalization_run_time] || nil }
+                normalization_run_time: status_info[:normalization_run_time] || nil,
+                pending: status_info[:pending] || false }
     end
 
     # Delete from the log any tests that passed.
@@ -60,16 +63,83 @@ module QaServer
     # Append a log to this log.
     # @param [ScenarioLog] the log to append to this log
     def append(other)
-      return unless other.present?
+      return if other.blank?
       @log += other.to_a
       @test_count += other.test_count
       @failure_count += other.failure_count
     end
 
     # @return selected scenario test results data as an array limited to the specified type or all scenarios if type is nil
-    def filter(type: nil)
+    # @example ungrouped
+    #   [ { type: :accuracy_test,
+    #       status: :PASS,
+    #       authority_name: 'CERL_LD4L_CACHE',
+    #       subauthority_name: 'imprint',
+    #       service: 'ld4l_cache',
+    #       action: 'search',
+    #       url: '/qa/search/linked_data/cerl_ld4l_cache/imprint?q=Plantin&maxRecords=8',
+    #       request_data: 'Plantin',
+    #       expected: 1,
+    #       actual: 1,
+    #       target: 'http://thesaurus.cerl.org/record/cni00007649',
+    #       err_message: '',
+    #       request_run_time: 0.032,
+    #       normalization_run_time: 0.011,
+    #       pending: false },
+    #       ... ]
+    # @example grouped
+    #   { CERL_LD4L_CACHE =
+    #       [ { type: :accuracy_test,
+    #           status: :PASS,
+    #           authority_name: 'CERL_LD4L_CACHE',
+    #           subauthority_name: 'imprint',
+    #           service: 'ld4l_cache',
+    #           action: 'search',
+    #           url: '/qa/search/linked_data/cerl_ld4l_cache/imprint?q=Plantin&maxRecords=8',
+    #           request_data: 'Plantin',
+    #           expected: 1,
+    #           actual: 1,
+    #           target: 'http://thesaurus.cerl.org/record/cni00007649',
+    #           err_message: '',
+    #           request_run_time: 0.032,
+    #           normalization_run_time: 0.011,
+    #           pending: false },
+    #           ... # all others for CERL_LD4L_CACHE
+    #       ],
+    #     CERL_NEW_LD4L_CACHE =
+    #       [ { type: :accuracy_test,
+    #           status: :PASS,
+    #           authority_name: 'CERL_NEW_LD4L_CACHE',
+    #           subauthority_name: 'imprint',
+    #           service: 'ld4l_cache',
+    #           action: 'search',
+    #           url: '/qa/search/linked_data/cerl_new_ld4l_cache/imprint?q=Plantin&maxRecords=8',
+    #           request_data: 'Plantin',
+    #           expected: 1,
+    #           actual: 1,
+    #           target: 'http://thesaurus.cerl.org/record/cni00007649',
+    #           err_message: '',
+    #           request_run_time: 0.022,
+    #           normalization_run_time: 0.011,
+    #           pending: false },
+    #           ... # all others for CERL_NEW_LD4L_CACHE
+    #       ]
+    #   }
+    def filter(type: nil, group: false)
+      return group_data(@log) if group && type.blank?
       return @log if type.blank?
-      @log.select { |entry| entry[:type] == type }
+      filtered_log = @log.select { |entry| entry[:type] == type }
+      group ? group_data(filtered_log) : filtered_log
+    end
+
+    def group_data(log)
+      grouped_data = {}
+      log.each do |datum|
+        auth_name = datum[:authority_name]
+        grouped_data[auth_name] = [] unless grouped_data.key? auth_name
+        grouped_data[auth_name] << datum
+      end
+      grouped_data
     end
 
     # @return the scenario test results data as an array

data/app/models/concerns/qa_server/performance_history_data_keys.rb

@@ -38,5 +38,13 @@ module QaServer
     HIGH_NORM = :high_normalization_ms
     HIGH_ACTN = :high_action_request_ms
     HIGH_FULL = :high_full_request_ms
+
+    SRC_BYTES = :data_raw_bytes_from_source
+    BPMS_RETR = :retrieve_bytes_per_ms
+    MSPB_RETR = :retrieve_ms_per_byte
+    BPMS_GRPH = :load_graph_bytes_per_ms
+    MSPB_GRPH = :load_graph_ms_per_byte
+    BPMS_NORM = :normalization_bytes_per_ms
+    MSPB_NORM = :normalization_ms_per_byte
   end
 end