qa_server 7.1.3 → 7.5.0

data/lib/qa_server/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module QaServer
-  VERSION = '7.1.3'
+  VERSION = '7.5.0'
 end

data/qa_server.gemspec

@@ -34,7 +34,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'better_errors' # provide debugging command line in
   spec.add_development_dependency 'binding_of_caller' # provides deep stack info used by better_errors
-  spec.add_development_dependency 'bixby', '~> 1.0.0' # rubocop styleguide
+  spec.add_development_dependency 'bixby', '~> 3.0.0' # rubocop styleguide
   # spec.add_development_dependency "capybara", '~> 3.29'
   # spec.add_development_dependency 'capybara-maleficent', '~> 0.3.0'
   # spec.add_development_dependency 'chromedriver-helper', '~> 2.1'

data/spec/feature/accuracy_spec.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+# rubocop:disable RSpec/InstanceVariable
+RSpec.describe 'Accuracy test' do # rubocop:disable RSpec/DescribeClass
+  before(:all) { WebMock.allow_net_connect! }
+  after(:all) { WebMock.disable_net_connect! }
+
+  let(:authority_list) { QaServer::AuthorityListerService.authorities_list }
+  let(:authority_name) { :CERL_LD4L_CACHE }
+
+  describe 'for authority:' do
+    @status_log = QaServer::ScenarioLogger.new
+    QaServer::AuthorityListerService.authorities_list.each do |authority_name| # rubocop:disable Style/MultilineIfModifier
+      QaServer::AuthorityValidatorService.run(authority_name: authority_name,
+                                              status_log: @status_log,
+                                              validation_type: QaServer::ScenarioValidator::VALIDATE_ACCURACY)
+    end unless ENV['TRAVIS']
+    @status_log.each do |test_result|
+      context "#{test_result[:authority_name]}:#{test_result[:subauthority_name]}:#{test_result[:request_data]}:" do
+        it "finds actual <= expected" do
+          pending 'test is known to fail' if test_result[:pending]
+          expect(test_result[:err_message]).to be_empty
+          expect(test_result[:request_data]).not_to be_empty
+          expect(test_result[:actual]).not_to be_nil
+          expect(test_result[:actual]).to be <= test_result[:expected]
+        end
+      end
+    end
+  end
+end
+# rubocop:enable RSpec/InstanceVariable

data/spec/spec_helper.rb

@@ -62,3 +62,7 @@ def load_fixture_file(fname)
     return f.read
   end
 end
+
+QaServer.config.suppress_performance_gathering = true
+QaServer.config.suppress_logging_performance_details = true
+QaServer.config.preferred_time_zone_name = 'Eastern Time (US & Canada)'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: qa_server
 version: !ruby/object:Gem::Version
-  version: 7.1.3
+  version: 7.5.0
 platform: ruby
 authors:
 - E. Lynette Rayle
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-22 00:00:00.000000000 Z
+date: 2020-12-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 3.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: deprecation
   requirement: !ruby/object:Gem::Requirement
@@ -340,11 +340,12 @@ files:
 - app/assets/stylesheets/qa_server/_usage.scss
 - app/cache_processors/concerns/qa_server/cache_keys.rb
 - app/cache_processors/qa_server/cache_expiry_service.rb
+- app/cache_processors/qa_server/job_id_cache.rb
 - app/cache_processors/qa_server/performance_cache.rb
-- app/cache_processors/qa_server/performance_daily_graph_cache.rb
 - app/cache_processors/qa_server/performance_datatable_cache.rb
-- app/cache_processors/qa_server/performance_hourly_graph_cache.rb
-- app/cache_processors/qa_server/performance_monthly_graph_cache.rb
+- app/cache_processors/qa_server/performance_day_graph_cache.rb
+- app/cache_processors/qa_server/performance_month_graph_cache.rb
+- app/cache_processors/qa_server/performance_year_graph_cache.rb
 - app/cache_processors/qa_server/scenario_history_cache.rb
 - app/cache_processors/qa_server/scenario_history_graph_cache.rb
 - app/cache_processors/qa_server/scenario_run_cache.rb
@@ -357,7 +358,12 @@ files:
 - app/controllers/qa_server/homepage_controller.rb
 - app/controllers/qa_server/monitor_status_controller.rb
 - app/controllers/qa_server/usage_controller.rb
+- app/jobs/qa_server/history_graph_job.rb
 - app/jobs/qa_server/monitor_tests_job.rb
+- app/jobs/qa_server/performance_day_graph_job.rb
+- app/jobs/qa_server/performance_month_graph_job.rb
+- app/jobs/qa_server/performance_per_byte_job.rb
+- app/jobs/qa_server/performance_year_graph_job.rb
 - app/loggers/qa_server/scenario_logger.rb
 - app/models/concerns/qa_server/performance_history_data_keys.rb
 - app/models/qa_server/authority_scenario.rb
@@ -373,7 +379,6 @@ files:
 - app/prepends/prepended_linked_data/find_term.rb
 - app/prepends/prepended_linked_data/search_query.rb
 - app/prepends/prepended_rdf/rdf_graph.rb
-- app/presenters/concerns/qa_server/monitor_status/gruff_graph.rb
 - app/presenters/concerns/qa_server/monitor_status/performance_datatable_behavior.rb
 - app/presenters/concerns/qa_server/monitor_status/performance_graph_behavior.rb
 - app/presenters/qa_server/authority_list_presenter.rb
@@ -384,6 +389,7 @@ files:
 - app/presenters/qa_server/monitor_status/performance_presenter.rb
 - app/presenters/qa_server/monitor_status_presenter.rb
 - app/presenters/qa_server/navmenu_presenter.rb
+- app/services/concerns/qa_server/gruff_graph.rb
 - app/services/qa_server/authority_lister_service.rb
 - app/services/qa_server/authority_loader_service.rb
 - app/services/qa_server/authority_validator_service.rb
@@ -393,6 +399,8 @@ files:
 - app/services/qa_server/performance_datatable_service.rb
 - app/services/qa_server/performance_graph_data_service.rb
 - app/services/qa_server/performance_graphing_service.rb
+- app/services/qa_server/performance_per_byte_calculator_service.rb
+- app/services/qa_server/performance_per_byte_data_service.rb
 - app/services/qa_server/scenarios_loader_service.rb
 - app/services/qa_server/time_period_service.rb
 - app/services/qa_server/time_service.rb
@@ -429,13 +437,18 @@ files:
 - lib/generators/qa_server/templates/config/authorities/linked_data/getty_aat_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/getty_tgn_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/getty_ulan_ld4l_cache.json
+- lib/generators/qa_server/templates/config/authorities/linked_data/isni_ld4l_cache.json
+- lib/generators/qa_server/templates/config/authorities/linked_data/ligatus_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/loc_direct.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/locdemographics_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/locgenres_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/locnames_ld4l_cache.json
+- lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo2_ld4l_cache.json
+- lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo3_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/locperformance_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/locsubjects_ld4l_cache.json
+- lib/generators/qa_server/templates/config/authorities/linked_data/locvocabs_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/mesh_nlm_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/nalt_direct.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/nalt_ld4l_cache.json
@@ -452,6 +465,8 @@ files:
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/getty_aat_ld4l_cache_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/getty_tgn_ld4l_cache_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/getty_ulan_ld4l_cache_validation.yml
+- lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/isni_ld4l_cache_validation.yml
+- lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/ligatus_ld4l_cache_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/loc_direct_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/loc_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locdemographics_ld4l_cache_validation.yml
@@ -460,6 +475,7 @@ files:
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_rwo_ld4l_cache_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locperformance_ld4l_cache_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locsubjects_ld4l_cache_validation.yml
+- lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locvocabs_ld4l_cache_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/mesh_nlm_ld4l_cache_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/nalt_direct_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/nalt_ld4l_cache_validation.yml
@@ -491,6 +507,7 @@ files:
 - qa_server.gemspec
 - spec/.gitignore
 - spec/cache_processors/qa_server/cache_expiry_service_spec.rb
+- spec/feature/accuracy_spec.rb
 - spec/lib/configuration_spec.rb
 - spec/lib/qa_server_spec.rb
 - spec/rails_helper.rb
@@ -505,7 +522,7 @@ homepage: http://github.com/LD4P/qa_server
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -520,14 +537,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14
-signing_key: 
+rubygems_version: 3.0.8
+signing_key:
 specification_version: 4
 summary: Authority Lookup Server
 test_files:
 - spec/.gitignore
 - spec/cache_processors/qa_server/cache_expiry_service_spec.rb
+- spec/feature/accuracy_spec.rb
 - spec/lib/configuration_spec.rb
 - spec/lib/qa_server_spec.rb
 - spec/rails_helper.rb

data/app/cache_processors/qa_server/performance_daily_graph_cache.rb

@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-# Generate graphs for the past 30 days using cached data.  Graphs are generated only if the cache has expired.
-module QaServer
-  class PerformanceDailyGraphCache
-    class_attribute :authority_list_class, :graph_data_service, :graphing_service
-    self.authority_list_class = QaServer::AuthorityListerService
-    self.graph_data_service = QaServer::PerformanceGraphDataService
-    self.graphing_service = QaServer::PerformanceGraphingService
-
-    class << self
-      include QaServer::CacheKeys
-      include QaServer::PerformanceHistoryDataKeys
-
-      # Generates graphs for the past 30 days for :search, :fetch, and :all actions for each authority.
-      # @param force [Boolean] if true, run the tests even if the cache hasn't expired; otherwise, use cache if not expired
-      def generate_graphs(force: false)
-        return unless QaServer::CacheExpiryService.cache_expired?(key: cache_key_for_force, force: force, next_expiry: next_expiry)
-        QaServer.config.monitor_logger.debug("(QaServer::PerformanceDailyGraphCache) - GENERATING daily performance graphs (force: #{force})")
-        generate_graphs_for_authorities
-      end
-
-      private
-
-        def generate_graphs_for_authorities
-          auths = authority_list_class.authorities_list
-          generate_graphs_for_authority(authority_name: ALL_AUTH) # generates graph for all authorities
-          auths.each { |authname| generate_graphs_for_authority(authority_name: authname) }
-        end
-
-        def generate_graphs_for_authority(authority_name:)
-          [SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
-            hash[action] = generate_30_day_graph(authority_name: authority_name, action: action)
-          end
-        end
-
-        def generate_30_day_graph(authority_name:, action:)
-          # real expiration or force caught by cache_expired?  So if we are here, either the cache has expired
-          # or force was requested.  We still expire the cache and use ttl to catch race conditions.
-          Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
-                            expires_in: next_expiry, race_condition_ttl: 1.hour, force: true) do
-            data = graph_data_service.calculate_last_30_days(authority_name: authority_name, action: action)
-            graphing_service.generate_daily_graph(authority_name: authority_name, action: action, data: data)
-            data
-          end
-        end
-
-        def cache_key_for_authority_action(authority_name:, action:)
-          "#{PERFORMANCE_GRAPH_DAILY_DATA_CACHE_KEY}--#{authority_name}--#{action}"
-        end
-
-        def cache_key_for_force
-          "#{PERFORMANCE_GRAPH_DAILY_DATA_CACHE_KEY}--force"
-        end
-
-        def next_expiry
-          QaServer::CacheExpiryService.cache_expiry
-        end
-    end
-  end
-end

data/app/cache_processors/qa_server/performance_hourly_graph_cache.rb

@@ -1,65 +0,0 @@
-# frozen_string_literal: true
-# Generate graphs for the past 24 hours using cached data.  The last hour of data is always calculated and all graphs
-# for are generated.
-module QaServer
-  class PerformanceHourlyGraphCache
-    class_attribute :authority_list_class, :graph_data_service, :graphing_service
-    self.authority_list_class = QaServer::AuthorityListerService
-    self.graph_data_service = QaServer::PerformanceGraphDataService
-    self.graphing_service = QaServer::PerformanceGraphingService
-
-    class << self
-      include QaServer::CacheKeys
-      include QaServer::PerformanceHistoryDataKeys
-
-      # Generates graphs for the past 24 hours for :search, :fetch, and :all actions for each authority.
-      # @param force [Boolean] if true, run the tests even if the cache hasn't expired; otherwise, use cache if not expired
-      def generate_graphs(force: false)
-        QaServer.config.monitor_logger.debug("(QaServer::PerformanceHourlyGraphCache) - GENERATING hourly performance graphs (force: #{force})")
-        QaServer.config.performance_cache.write_all
-        generate_graphs_for_authorities(force: force)
-      end
-
-      private
-
-        def generate_graphs_for_authorities(force:)
-          auths = authority_list_class.authorities_list
-          generate_graphs_for_authority(authority_name: ALL_AUTH, force: force) # generates graph for all authorities
-          auths.each { |authname| generate_graphs_for_authority(authority_name: authname, force: force) }
-        end
-
-        def generate_graphs_for_authority(authority_name:, force:)
-          [SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
-            hash[action] = generate_24_hour_graph(authority_name: authority_name, action: action, force: force)
-          end
-        end
-
-        def generate_24_hour_graph(authority_name:, action:, force:)
-          graph_created = false
-          data = Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
-                                   expires_in: QaServer::TimeService.current_time.end_of_hour - QaServer::TimeService.current_time,
-                                   race_condition_ttl: 1.hour, force: force) do
-            data = graph_data_service.calculate_last_24_hours(authority_name: authority_name, action: action)
-            graphing_service.generate_hourly_graph(authority_name: authority_name, action: action, data: data)
-            graph_created = true
-            data
-          end
-          regen_last_hour_and_graph(authority_name: authority_name, action: action, data: data) unless graph_created
-        end
-
-        def regen_last_hour_and_graph(authority_name:, action:, data:)
-          Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
-                            expires_in: QaServer::TimeService.current_time.end_of_hour - QaServer::TimeService.current_time,
-                            race_condition_ttl: 1.hour, force: true) do
-            data = graph_data_service.recalculate_last_hour(authority_name: authority_name, action: action, averages: data)
-            graphing_service.generate_hourly_graph(authority_name: authority_name, action: action, data: data)
-            data
-          end
-        end
-
-        def cache_key_for_authority_action(authority_name:, action:)
-          "#{PERFORMANCE_GRAPH_HOURLY_DATA_CACHE_KEY}--#{authority_name}--#{action}"
-        end
-    end
-  end
-end

data/app/cache_processors/qa_server/performance_monthly_graph_cache.rb

@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-# Generate graphs for the past 12 months using cached data.  Graphs are generated only if the cache has expired.
-module QaServer
-  class PerformanceMonthlyGraphCache
-    class_attribute :authority_list_class, :graph_data_service, :graphing_service
-    self.authority_list_class = QaServer::AuthorityListerService
-    self.graph_data_service = QaServer::PerformanceGraphDataService
-    self.graphing_service = QaServer::PerformanceGraphingService
-
-    class << self
-      include QaServer::CacheKeys
-      include QaServer::PerformanceHistoryDataKeys
-
-      # Generates graphs for the past 30 days for :search, :fetch, and :all actions for each authority.
-      # @param force [Boolean] if true, run the tests even if the cache hasn't expired; otherwise, use cache if not expired
-      def generate_graphs(force: false)
-        return unless QaServer::CacheExpiryService.cache_expired?(key: cache_key_for_force, force: force, next_expiry: next_expiry)
-        QaServer.config.monitor_logger.debug("(QaServer::PerformanceMonthlyGraphCache) - GENERATING monthly performance graphs (force: #{force})")
-        generate_graphs_for_authorities
-      end
-
-      private
-
-        def generate_graphs_for_authorities
-          auths = authority_list_class.authorities_list
-          generate_graphs_for_authority(authority_name: ALL_AUTH) # generates graph for all authorities
-          auths.each { |authname| generate_graphs_for_authority(authority_name: authname) }
-        end
-
-        def generate_graphs_for_authority(authority_name:)
-          [SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
-            hash[action] = generate_12_month_graph(authority_name: authority_name, action: action)
-          end
-        end
-
-        def generate_12_month_graph(authority_name:, action:)
-          # real expiration or force caught by cache_expired?  So if we are here, either the cache has expired
-          # or force was requested.  We still expire the cache and use ttl to catch race conditions.
-          Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
-                            expires_in: next_expiry, race_condition_ttl: 1.hour, force: true) do
-            data = graph_data_service.calculate_last_12_months(authority_name: authority_name, action: action)
-            graphing_service.generate_monthly_graph(authority_name: authority_name, action: action, data: data)
-            data
-          end
-        end
-
-        def cache_key_for_authority_action(authority_name:, action:)
-          "#{PERFORMANCE_GRAPH_MONTHLY_DATA_CACHE_KEY}--#{authority_name}--#{action}"
-        end
-
-        def cache_key_for_force
-          "#{PERFORMANCE_GRAPH_MONTHLY_DATA_CACHE_KEY}--force"
-        end
-
-        def next_expiry
-          QaServer::CacheExpiryService.cache_expiry
-        end
-    end
-  end
-end

data/app/presenters/concerns/qa_server/monitor_status/gruff_graph.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-require 'fileutils'
-require 'gruff'
-
-# This module include provides graph methods used by all monitor status presenters working with graphs
-module QaServer::MonitorStatus
-  module GruffGraph
-    private
-
-      def graph_relative_path
-        File.join('qa_server', 'charts')
-      end
-
-      def graph_full_path(graph_filename)
-        path = Rails.root.join('app', 'assets', 'images', graph_relative_path)
-        FileUtils.mkdir_p path
-        File.join(path, graph_filename)
-      end
-  end
-end