qa_server 6.1.0 → 6.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3f5c0789b598b78ba1ac24e8070d989c92147bad
-  data.tar.gz: a3d686757d3cb4f9279c505416c6c3a54edbc00c
+  metadata.gz: b602ac44639871d5021eb47493edf93fb8abdc25
+  data.tar.gz: bf9e6a5ece3786edab98ed79d1fc3edb8e54f8b7
 SHA512:
-  metadata.gz: fc7340db3cc8a3aaf1d20449608427b57c7aa03a0d096b67b093330c92e97b4af7035acd20833bcf74e440056eebd7a6b37bf5224c95686d14294812612ad447
-  data.tar.gz: d7e0ae7f314ec9c7fb9d16783947a2a9cb5d7dea10388b6ad777c0da8a2e9c5ca035511bd519ed3b4bcb3314e6c782a6901983459a5872cdea0980c0333a6d56
+  metadata.gz: ec1fec69f78b763fc7012c92118fddeb6e0f6ad98d53d9ae80d959f80f0c1373c435626452644294ecbcf1b1843ac37517af58a71e04ece333a9d14afa99486f
+  data.tar.gz: d4e87a9c8157f8d03b3fdbc26ded7698f294cee41f9c7a2222355db878c26ab7e7d121d32dda69fbf1660f5b7942de7ec05688431a364b1e06f22b9b73967598

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+### 6.2.0 (2020-02-17)
+
+* use authentication for refreshing monitor tests
+* add performance cache logger
+* exceeding max performance cache size flushing the cache
+
 ### 6.1.0 (2020-02-17)
 
 * change historical summary to show number of days instead of number of tests (original intent)

data/app/controllers/qa_server/check_status_controller.rb

@@ -13,6 +13,7 @@ module QaServer
 
     # Sets up presenter with data to display in the UI
     def index
+      log_header
       validate(authorities_to_validate, validation_type)
       @presenter = presenter_class.new(authorities_list: authorities_list,
                                        connection_status_data: connection_status_data_from_log,
@@ -38,5 +39,10 @@ module QaServer
         return @authority_name if @authority_name.present?
         @authority_name = params.key?(:authority) ? params[:authority].downcase : nil
       end
+
+      def log_header
+        QaServer.config.performance_cache_logger.debug("-------------------------------------  check status  ---------------------------------")
+        QaServer.config.performance_cache_logger.debug("(#{self.class}##{__method__}) check status page request (authority_name # #{authority_name})")
+      end
   end
 end

data/app/controllers/qa_server/monitor_status_controller.rb

@@ -21,11 +21,13 @@ module QaServer
     def index
       log_header
       latest_test_run
+      commit_cache if commit_cache?
       @presenter = presenter_class.new(current_summary: latest_summary,
                                        current_failure_data: latest_failures,
                                        historical_summary_data: historical_data,
                                        performance_data: performance_table_data)
       update_performance_graphs
+      QaServer.config.monitor_logger.debug("(#{self.class}##{__method__}) DONE rendering")
       render 'index', status: :internal_server_error if latest_summary.failing_authority_count.positive?
     end
 
@@ -40,7 +42,7 @@ module QaServer
       # @see #latest_test_run_from_temp_cache
       def latest_test_run_from_cache
         Rails.cache.fetch("#{self.class}/#{__method__}", expires_in: QaServer::MonitorCacheService.cache_expiry, race_condition_ttl: 5.minutes, force: refresh_tests?) do
-          QaServer.config.monitor_logger.info("(#{self.class}##{__method__}) get latest run of monitoring tests - cache expired or refresh requested (force: #{refresh_tests?})")
+          QaServer.config.monitor_logger.debug("(#{self.class}##{__method__}) get latest run of monitoring tests - cache expired or refresh requested (force: #{refresh_tests?})")
           QaServer::MonitorTestsJob.perform_later
           scenario_run_registry_class.latest_run
         end
@@ -80,7 +82,7 @@ module QaServer
       end
 
       def refresh?
-        params.key? :refresh
+        params.key?(:refresh) && validate_auth_reload_token("refresh status")
       end
 
       def refresh_all?
@@ -103,10 +105,28 @@ module QaServer
         refresh_all? || params[:refresh].casecmp?('performance')
       end
 
+      def commit_cache?
+        params.key?(:commit) && validate_auth_reload_token("commit cache")
+      end
+
+      def commit_cache
+        QaServer.config.performance_cache.write_all
+      end
+
+      def validate_auth_reload_token(action)
+        token = params.key?(:auth_token) ? params[:auth_token] : nil
+        valid = Qa.config.valid_authority_reload_token?(token)
+        return true if valid
+        msg = "Permission denied. Unable to #{action}."
+        logger.warn msg
+        flash.now[:error] = msg
+        false
+      end
+
       def log_header
         QaServer.config.monitor_logger.debug("-------------------------------------  monitor status  ---------------------------------")
-        QaServer.config.monitor_logger.info("(#{self.class}##{__method__}) monitor status page request (refresh_tests? # #{refresh_tests?}, " \
-                                         "refresh_history? # #{refresh_history?}, refresh_performance? # #{refresh_performance?})")
+        QaServer.config.monitor_logger.debug("(#{self.class}##{__method__}) monitor status page request (refresh_tests? # #{refresh_tests?}, " \
+                                             "refresh_history? # #{refresh_history?}, refresh_performance? # #{refresh_performance?})")
       end
   end
 end

data/app/jobs/qa_server/monitor_tests_job.rb

@@ -9,9 +9,8 @@ module QaServer
     class_attribute :scenario_run_registry_class
     self.scenario_run_registry_class = QaServer::ScenarioRunRegistry
 
-    # def perform(job_id:)
     def perform
-      Rails.cache.fetch("QaServer::MonitorTestsController/latest_test_run_from_cache", expires_in: QaServer::MonitorCacheService.cache_expiry, race_condition_ttl: 5.minutes, force: true) do
+      Rails.cache.fetch("QaServer::MonitorStatusController/latest_test_run_from_cache", expires_in: QaServer::MonitorCacheService.cache_expiry, race_condition_ttl: 5.minutes, force: true) do
         job_id = SecureRandom.uuid
         monitor_tests_job_id = job_id unless monitor_tests_job_id
         run_tests if monitor_tests_job_id == job_id # avoid race conditions
@@ -22,11 +21,11 @@ module QaServer
     private
 
       def run_tests
-        QaServer.config.monitor_logger.info("(#{self.class}##{__method__}-#{job_id}) RUNNING monitoring tests")
+        QaServer.config.monitor_logger.debug("(#{self.class}##{__method__}-#{job_id}) RUNNING monitoring tests")
         validate(authorities_list)
         log_results(authorities_list, status_log.to_a)
         scenario_run_registry_class.save_run(scenarios_results: status_log.to_a)
-        QaServer.config.monitor_logger.info("(#{self.class}##{__method__}-#{job_id}) COMPLETED monitoring tests")
+        QaServer.config.monitor_logger.debug("(#{self.class}##{__method__}-#{job_id}) COMPLETED monitoring tests")
         reset_monitor_tests_job_id
       end
 

data/app/models/qa_server/performance_cache.rb

@@ -21,32 +21,47 @@ module QaServer
       @cache[id] = entry.merge(updates)
     end
 
+    def complete_entry(id:)
+      log(id: id)
+      QaServer.config.performance_cache_logger.debug("#{self.class}##{__method__} - id: #{id}   cache memory: #{ObjectSpace.memsize_of @cache}")
+      write_all if ObjectSpace.memsize_of(@cache) > QaServer.config.max_performance_cache_size
+    end
+
     def destroy(id)
-      @cache.delete(id)
+      @cache.delete(id) # WARNING: doesn't change the size of the cache
     end
 
     def write_all
-      size_before = @cache.size
-      @cache.each do |id, entry|
+      cache_to_write = swap_cache_hash
+      size_before = cache_to_write.size
+      cache_to_write.each do |id, entry|
         next if incomplete? entry
         QaServer::PerformanceHistory.create(dt_stamp: entry[:dt_stamp], authority: entry[:authority],
                                             action: entry[:action], action_time_ms: entry[:action_time_ms],
                                             size_bytes: entry[:size_bytes], retrieve_time_ms: entry[:retrieve_time_ms],
                                             graph_load_time_ms: entry[:graph_load_time_ms],
                                             normalization_time_ms: entry[:normalization_time_ms])
-        @cache.delete(id)
+        cache_to_write.delete(id)
       end
-      log_write_all("(#{self.class}##{__method__})", size_before, @cache.size)
-    end
-
-    def log(id:)
-      return if QaServer.config.suppress_logging_performance_datails
-      Rails.logger.debug("*** performance data for id: #{id} ***")
-      Rails.logger.debug(@cache[id].to_yaml)
+      log_write_all("(#{self.class}##{__method__})", size_before, cache_to_write.size)
+      cache_to_write = nil # free cache for garbage collection
     end
 
     private
 
+      def swap_cache_hash
+        cache_to_write = @cache
+        @cache = {} # reset main cache so new items after write begins are cached in the main cache
+        QaServer.config.performance_cache_logger.debug("#{self.class}##{__method__} - cache memory BEFORE write: #{ObjectSpace.memsize_of(cache_to_write)}")
+        cache_to_write
+      end
+
+      def log(id:)
+        return if QaServer.config.suppress_logging_performance_datails
+        Rails.logger.debug("*** performance data for id: #{id} ***")
+        Rails.logger.debug(@cache[id].to_yaml)
+      end
+
       def incomplete?(entry)
         required_keys.each { |k| return true unless entry.key? k }
         false
@@ -65,11 +80,12 @@ module QaServer
 
       def log_write_all(prefix, size_before, cache_size)
         if size_before.positive?
-          QaServer.config.monitor_logger.warn("#{prefix} 0 of #{size_before} performance data records were saved") if size_before == cache_size
-          QaServer.config.monitor_logger.info("#{prefix} #{size_before - cache_size} of #{size_before} performance data records were saved") if size_before > cache_size
+          QaServer.config.performance_cache_logger.debug("#{prefix} 0 of #{size_before} performance data records were saved") if size_before == cache_size
+          QaServer.config.performance_cache_logger.debug("#{prefix} #{size_before - cache_size} of #{size_before} performance data records were saved") if size_before > cache_size
         else
-          QaServer.config.monitor_logger.info("#{prefix} 0 of 0 performance data records were saved")
+          QaServer.config.performance_cache_logger.debug("#{prefix} 0 of 0 performance data records were saved")
         end
+        QaServer.config.performance_cache_logger.debug("#{prefix} - cache memory AFTER write: #{ObjectSpace.memsize_of @cache}")
       end
   end
 end

data/app/models/qa_server/scenario_run_history.rb

@@ -47,7 +47,7 @@ module QaServer
       #   * total_scenario_count: 159,
       def run_summary(scenario_run:, force: false)
         Rails.cache.fetch("QaServer::ScenarioRunHistory/#{__method__}", expires_in: QaServer::MonitorCacheService.cache_expiry, race_condition_ttl: 1.minute, force: force) do
-          QaServer.config.monitor_logger.info("(QaServer::ScenarioRunHistory##{__method__}) - CALCULATING summary of latest run - cache expired or refresh requested (force: #{force})")
+          QaServer.config.monitor_logger.debug("(QaServer::ScenarioRunHistory##{__method__}) - CALCULATING summary of latest run - cache expired or refresh requested (force: #{force})")
           status = status_counts_in_run(run_id: scenario_run.id)
           summary_class.new(run_id: scenario_run.id,
                             run_dt_stamp: scenario_run.dt_stamp,
@@ -130,7 +130,7 @@ module QaServer
       def run_failures(run_id:, force: false)
         return [] unless run_id
         Rails.cache.fetch("QaServer::ScenarioRunHistory/#{__method__}", expires_in: QaServer::MonitorCacheService.cache_expiry, race_condition_ttl: 1.minute, force: force) do
-          QaServer.config.monitor_logger.info("(QaServer::ScenarioRunHistory##{__method__}) - finding failures in latest run - cache expired or refresh requested (force: #{force})")
+          QaServer.config.monitor_logger.debug("(QaServer::ScenarioRunHistory##{__method__}) - finding failures in latest run - cache expired or refresh requested (force: #{force})")
           QaServer::ScenarioRunHistory.where(scenario_run_registry_id: run_id).where.not(status: :good).to_a
         end
       end
@@ -142,7 +142,7 @@ module QaServer
       #     'geonames_ld4l_cache' => { good: 32, bad: 1 } }
       def historical_summary(force: false)
         Rails.cache.fetch("QaServer::ScenarioRunHistory/#{__method__}", expires_in: QaServer::MonitorCacheService.cache_expiry, race_condition_ttl: 1.minute, force: force) do
-          QaServer.config.monitor_logger.info("(QaServer::ScenarioRunHistory##{__method__}) - CALCULATING authority connection history - cache expired or refresh requested (force: #{force})")
+          QaServer.config.monitor_logger.debug("(QaServer::ScenarioRunHistory##{__method__}) - CALCULATING authority connection history - cache expired or refresh requested (force: #{force})")
           days_good = count_days(:good)
           days_bad = count_days(:bad)
           days_unknown = count_days(:unknown)

data/app/prepends/prepended_linked_data/find_term.rb

@@ -35,7 +35,7 @@ module PrependedLinkedData::FindTerm
                   retrieve_plus_graph_load_time_ms: full_results[:performance][:fetch_time_s] * 1000,
                   normalization_time_ms: full_results[:performance][:normalization_time_s] * 1000 }
       QaServer.config.performance_cache.update(id: @phid, updates: updates)
-      QaServer.config.performance_cache.log(id: @phid)
+      QaServer.config.performance_cache.complete_entry(id: @phid)
     end
 
     # Override to append performance history record id into the URL to allow access to the record in RDF::Graph

data/app/prepends/prepended_linked_data/search_query.rb

@@ -35,7 +35,7 @@ module PrependedLinkedData::SearchQuery
                   retrieve_plus_graph_load_time_ms: full_results[:performance][:fetch_time_s] * 1000,
                   normalization_time_ms: full_results[:performance][:normalization_time_s] * 1000 }
       QaServer.config.performance_cache.update(id: @phid, updates: updates)
-      QaServer.config.performance_cache.log(id: @phid)
+      QaServer.config.performance_cache.complete_entry(id: @phid)
     end
 
     # Override to append performance history record id into the URL to allow access to the record in RDF::Graph

data/app/services/qa_server/performance_datatable_service.rb

@@ -26,7 +26,7 @@ module QaServer
       #   }
       def calculate_datatable_data(force:)
         Rails.cache.fetch("QaServer::PerformanceDatatableService/#{__method__}", expires_in: QaServer::MonitorCacheService.cache_expiry, race_condition_ttl: 5.minutes, force: force) do
-          QaServer.config.monitor_logger.info("(QaServer::PerformanceDatatableService##{__method__}) - CALCULATING performance datatable stats - cache expired or refresh requested (force: #{force})")
+          QaServer.config.monitor_logger.debug("(QaServer::PerformanceDatatableService##{__method__}) - CALCULATING performance datatable stats - cache expired or refresh requested (force: #{force})")
           data = {}
           auths = authority_list_class.authorities_list
           data[ALL_AUTH] = datatable_data_for_authority

data/app/services/qa_server/performance_graph_data_service.rb

@@ -49,7 +49,7 @@ module QaServer
         data = {}
         auths = authority_list_class.authorities_list
         calculate_all = force || cache_expired?
-        QaServer.config.monitor_logger.info("(QaServer::PerformanceGraphDataService##{__method__}) - CALCULATING performance graph data (calculate_all: #{calculate_all})")
+        QaServer.config.monitor_logger.debug("(QaServer::PerformanceGraphDataService##{__method__}) - CALCULATING performance graph data (calculate_all: #{calculate_all})")
         data[ALL_AUTH] = graph_data_for_authority(force: force, calculate_all: calculate_all)
         auths.each { |auth_name| data[auth_name] = graph_data_for_authority(authority_name: auth_name, force: force, calculate_all: calculate_all) }
         data

data/app/services/qa_server/performance_graphing_service.rb

@@ -12,7 +12,7 @@ module QaServer
       # @param performance_data [Hash] hash of all performance data for all authorities
       # @see QaServer:PerformanceHistory
       def create_performance_graphs(performance_data:)
-        QaServer.config.monitor_logger.info("(QaServer::PerformanceGraphingService##{__method__}) - generating graphs")
+        QaServer.config.monitor_logger.debug("(QaServer::PerformanceGraphingService##{__method__}) - generating graphs")
         performance_data.each_key do |auth_name|
           create_graphs_for_authority(performance_data, auth_name.to_sym, :search)
           create_graphs_for_authority(performance_data, auth_name.to_sym, :fetch)

data/lib/generators/qa_server/templates/config/initializers/qa_server.rb

@@ -82,4 +82,20 @@ QaServer.config do |config|
   # by QaServer and can eat up logging realestate.  To suppress the logging of details, set this config to true.
   # @param [Boolean] do not log performance data details when true (defaults to false for backward compatibitily)
   # config.suppress_logging_performance_datails = false
+
+  # Maximum amount of memory the performance cache can occupy before it is written to the database.
+  # @param [Integer] maximum size of performance cache before flushing
+  # config.max_performance_cache_size = 32.megabytes
+
+  # Enable/Disable logging of performance cache
+  # Uncomment one of the lines below to enable or disable performance cache logging.  NOTE: By default, loggers follow the
+  # default levels for Rails loggers (i.e. enabled for development, disabled for production.)
+  # config.enable_performance_cache_logging
+  # config.disable_performance_cache_logging
+
+  # Enable/Disable logging of monitoring process
+  # Uncomment one of the lines below to enable or disable monitoring process logging.  NOTE: By default, loggers follow the
+  # default levels for Rails loggers (i.e. enabled for development, disabled for production.)
+  # config.enable_monitor_status_logging
+  # config.disable_monitor_status_logging
 end

data/lib/qa_server/configuration.rb

@@ -201,12 +201,43 @@ module QaServer
       @suppress_logging_performance_datails ||= false
     end
 
+    # Maximum amount of memory the performance cache can occupy before it is written to the database.
+    # @param [Integer] maximum size of performance cache before flushing
+    attr_writer :max_performance_cache_size
+    def max_performance_cache_size
+      @max_performance_cache_size ||= 32.megabytes
+    end
+
     # For internal use only
-    # TODO: consider refactor performance caching to use Rails cache
     def performance_cache
       @performance_cache ||= QaServer::PerformanceCache.new
     end
 
+    # Enable logging of performance cache
+    def enable_performance_cache_logging
+      performance_cache_logger.level = Logger::DEBUG
+    end
+
+    # Disable logging of performance cache
+    def disable_performance_cache_logging
+      performance_cache_logger.level = Logger::INFO
+    end
+
+    # For internal use only
+    def performance_cache_logger
+      @performance_cache_logger ||= Logger.new(ENV['PERFORMANCE_CACHE_LOG_PATH'] || File.join("log", "performance_cache.log"))
+    end
+
+    # Enable logging of monitoring process
+    def enable_monitor_status_logging
+      monitor_logger.level = Logger::DEBUG
+    end
+
+    # Disable logging of performance cache
+    def disable_monitor_status_logging
+      monitor_logger.level = Logger::INFO
+    end
+
     # For internal use only
     def monitor_logger
       @monitor_logger ||= Logger.new(ENV['MONITOR_LOG_PATH'] || File.join("log", "monitor.log"))

data/lib/qa_server/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module QaServer
-  VERSION = '6.1.0'
+  VERSION = '6.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: qa_server
 version: !ruby/object:Gem::Version
-  version: 6.1.0
+  version: 6.2.0
 platform: ruby
 authors:
 - E. Lynette Rayle
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-17 00:00:00.000000000 Z
+date: 2020-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails