qa_server 7.2.0 → 7.5.1

data/lib/qa_server/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module QaServer
-  VERSION = '7.2.0'
+  VERSION = '7.5.1'
 end

data/qa_server.gemspec

@@ -34,13 +34,14 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'better_errors' # provide debugging command line in
   spec.add_development_dependency 'binding_of_caller' # provides deep stack info used by better_errors
-  spec.add_development_dependency 'bixby', '~> 1.0.0' # rubocop styleguide
+  spec.add_development_dependency 'bixby', '~> 3.0.0' # rubocop styleguide
   # spec.add_development_dependency "capybara", '~> 3.29'
   # spec.add_development_dependency 'capybara-maleficent', '~> 0.3.0'
   # spec.add_development_dependency 'chromedriver-helper', '~> 2.1'
   spec.add_development_dependency 'deprecation'
   spec.add_development_dependency 'engine_cart', '~> 2.0'
   spec.add_development_dependency "factory_bot", '~> 4.4'
+  spec.add_development_dependency 'i18n-tasks'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'sqlite3'
   spec.add_development_dependency 'rails-controller-testing', '~> 1'

data/spec/feature/accuracy_spec.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+# rubocop:disable RSpec/InstanceVariable
+RSpec.describe 'Accuracy test' do # rubocop:disable RSpec/DescribeClass
+  before(:all) { WebMock.allow_net_connect! }
+  after(:all) { WebMock.disable_net_connect! }
+
+  let(:authority_list) { QaServer::AuthorityListerService.authorities_list }
+  let(:authority_name) { :CERL_LD4L_CACHE }
+
+  describe 'for authority:' do
+    @status_log = QaServer::ScenarioLogger.new
+    QaServer::AuthorityListerService.authorities_list.each do |authority_name| # rubocop:disable Style/MultilineIfModifier
+      QaServer::AuthorityValidatorService.run(authority_name: authority_name,
+                                              status_log: @status_log,
+                                              validation_type: QaServer::ScenarioValidator::VALIDATE_ACCURACY)
+    end unless ENV['TRAVIS']
+    @status_log.each do |test_result|
+      context "#{test_result[:authority_name]}:#{test_result[:subauthority_name]}:#{test_result[:request_data]}:" do
+        it "finds actual <= expected" do
+          pending 'test is known to fail' if test_result[:pending]
+          expect(test_result[:err_message]).to be_empty
+          expect(test_result[:request_data]).not_to be_empty
+          expect(test_result[:actual]).not_to be_nil
+          expect(test_result[:actual]).to be <= test_result[:expected]
+        end
+      end
+    end
+  end
+end
+# rubocop:enable RSpec/InstanceVariable

data/spec/i18n_spec.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'i18n/tasks'
+
+RSpec.describe I18n do
+  let(:i18n) { I18n::Tasks::BaseTask.new }
+  let(:missing_keys) { i18n.missing_keys }
+  let(:unused_keys) { i18n.unused_keys }
+  let(:inconsistent_interpolations) { i18n.inconsistent_interpolations }
+
+  it 'does not have missing keys' do
+    expect(missing_keys).to be_empty,
+                            "Missing #{missing_keys.leaves.count} i18n keys, run `i18n-tasks missing' to show them"
+  end
+
+  it 'does not have unused keys' do
+    pending 'need to explore unused keys further'
+    expect(unused_keys).to be_empty,
+                           "#{unused_keys.leaves.count} unused i18n keys, run `i18n-tasks unused' to show them"
+  end
+
+  it 'files are normalized' do # rubocop:disable RSpec/ExampleLength
+    pending 'need to explore further normalization process'
+    non_normalized = i18n.non_normalized_paths
+    error_message = "The following files need to be normalized:\n" \
+                    "#{non_normalized.map { |path| "  #{path}" }.join("\n")}\n" \
+                    "Please run `i18n-tasks normalize' to fix"
+    expect(non_normalized).to be_empty, error_message
+  end
+
+  it 'does not have inconsistent interpolations' do
+    error_message = "#{inconsistent_interpolations.leaves.count} i18n keys have inconsistent interpolations.\n" \
+                    "Run `i18n-tasks check-consistent-interpolations' to show them"
+    expect(inconsistent_interpolations).to be_empty, error_message
+  end
+end

data/spec/spec_helper.rb

@@ -62,3 +62,7 @@ def load_fixture_file(fname)
     return f.read
   end
 end
+
+QaServer.config.suppress_performance_gathering = true
+QaServer.config.suppress_logging_performance_details = true
+QaServer.config.preferred_time_zone_name = 'Eastern Time (US & Canada)'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: qa_server
 version: !ruby/object:Gem::Version
-  version: 7.2.0
+  version: 7.5.1
 platform: ruby
 authors:
 - E. Lynette Rayle
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-23 00:00:00.000000000 Z
+date: 2020-12-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 3.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: deprecation
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +164,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.4'
+- !ruby/object:Gem::Dependency
+  name: i18n-tasks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -340,11 +354,12 @@ files:
 - app/assets/stylesheets/qa_server/_usage.scss
 - app/cache_processors/concerns/qa_server/cache_keys.rb
 - app/cache_processors/qa_server/cache_expiry_service.rb
+- app/cache_processors/qa_server/job_id_cache.rb
 - app/cache_processors/qa_server/performance_cache.rb
-- app/cache_processors/qa_server/performance_daily_graph_cache.rb
 - app/cache_processors/qa_server/performance_datatable_cache.rb
-- app/cache_processors/qa_server/performance_hourly_graph_cache.rb
-- app/cache_processors/qa_server/performance_monthly_graph_cache.rb
+- app/cache_processors/qa_server/performance_day_graph_cache.rb
+- app/cache_processors/qa_server/performance_month_graph_cache.rb
+- app/cache_processors/qa_server/performance_year_graph_cache.rb
 - app/cache_processors/qa_server/scenario_history_cache.rb
 - app/cache_processors/qa_server/scenario_history_graph_cache.rb
 - app/cache_processors/qa_server/scenario_run_cache.rb
@@ -357,7 +372,12 @@ files:
 - app/controllers/qa_server/homepage_controller.rb
 - app/controllers/qa_server/monitor_status_controller.rb
 - app/controllers/qa_server/usage_controller.rb
+- app/jobs/qa_server/history_graph_job.rb
 - app/jobs/qa_server/monitor_tests_job.rb
+- app/jobs/qa_server/performance_day_graph_job.rb
+- app/jobs/qa_server/performance_month_graph_job.rb
+- app/jobs/qa_server/performance_per_byte_job.rb
+- app/jobs/qa_server/performance_year_graph_job.rb
 - app/loggers/qa_server/scenario_logger.rb
 - app/models/concerns/qa_server/performance_history_data_keys.rb
 - app/models/qa_server/authority_scenario.rb
@@ -393,6 +413,8 @@ files:
 - app/services/qa_server/performance_datatable_service.rb
 - app/services/qa_server/performance_graph_data_service.rb
 - app/services/qa_server/performance_graphing_service.rb
+- app/services/qa_server/performance_per_byte_calculator_service.rb
+- app/services/qa_server/performance_per_byte_data_service.rb
 - app/services/qa_server/scenarios_loader_service.rb
 - app/services/qa_server/time_period_service.rb
 - app/services/qa_server/time_service.rb
@@ -410,6 +432,7 @@ files:
 - app/views/qa_server/monitor_status/index.html.erb
 - app/views/qa_server/usage/index.html.erb
 - app/views/shared/_footer.html.erb
+- config/i18n-tasks.yml
 - config/locales/qa_server.en.yml
 - config/routes.rb
 - lib/generators/qa_server/assets_generator.rb
@@ -429,13 +452,18 @@ files:
 - lib/generators/qa_server/templates/config/authorities/linked_data/getty_aat_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/getty_tgn_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/getty_ulan_ld4l_cache.json
+- lib/generators/qa_server/templates/config/authorities/linked_data/isni_ld4l_cache.json
+- lib/generators/qa_server/templates/config/authorities/linked_data/ligatus_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/loc_direct.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/locdemographics_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/locgenres_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/locnames_ld4l_cache.json
+- lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo2_ld4l_cache.json
+- lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo3_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/locperformance_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/locsubjects_ld4l_cache.json
+- lib/generators/qa_server/templates/config/authorities/linked_data/locvocabs_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/mesh_nlm_ld4l_cache.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/nalt_direct.json
 - lib/generators/qa_server/templates/config/authorities/linked_data/nalt_ld4l_cache.json
@@ -452,6 +480,8 @@ files:
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/getty_aat_ld4l_cache_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/getty_tgn_ld4l_cache_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/getty_ulan_ld4l_cache_validation.yml
+- lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/isni_ld4l_cache_validation.yml
+- lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/ligatus_ld4l_cache_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/loc_direct_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/loc_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locdemographics_ld4l_cache_validation.yml
@@ -460,6 +490,7 @@ files:
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_rwo_ld4l_cache_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locperformance_ld4l_cache_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locsubjects_ld4l_cache_validation.yml
+- lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locvocabs_ld4l_cache_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/mesh_nlm_ld4l_cache_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/nalt_direct_validation.yml
 - lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/nalt_ld4l_cache_validation.yml
@@ -491,6 +522,8 @@ files:
 - qa_server.gemspec
 - spec/.gitignore
 - spec/cache_processors/qa_server/cache_expiry_service_spec.rb
+- spec/feature/accuracy_spec.rb
+- spec/i18n_spec.rb
 - spec/lib/configuration_spec.rb
 - spec/lib/qa_server_spec.rb
 - spec/rails_helper.rb
@@ -505,7 +538,7 @@ homepage: http://github.com/LD4P/qa_server
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -520,14 +553,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14
-signing_key: 
+rubygems_version: 3.0.8
+signing_key:
 specification_version: 4
 summary: Authority Lookup Server
 test_files:
 - spec/.gitignore
 - spec/cache_processors/qa_server/cache_expiry_service_spec.rb
+- spec/feature/accuracy_spec.rb
+- spec/i18n_spec.rb
 - spec/lib/configuration_spec.rb
 - spec/lib/qa_server_spec.rb
 - spec/rails_helper.rb

data/app/cache_processors/qa_server/performance_daily_graph_cache.rb

@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-# Generate graphs for the past 30 days using cached data.  Graphs are generated only if the cache has expired.
-module QaServer
-  class PerformanceDailyGraphCache
-    class_attribute :authority_list_class, :graph_data_service, :graphing_service
-    self.authority_list_class = QaServer::AuthorityListerService
-    self.graph_data_service = QaServer::PerformanceGraphDataService
-    self.graphing_service = QaServer::PerformanceGraphingService
-
-    class << self
-      include QaServer::CacheKeys
-      include QaServer::PerformanceHistoryDataKeys
-
-      # Generates graphs for the past 30 days for :search, :fetch, and :all actions for each authority.
-      # @param force [Boolean] if true, run the tests even if the cache hasn't expired; otherwise, use cache if not expired
-      def generate_graphs(force: false)
-        return unless QaServer::CacheExpiryService.cache_expired?(key: cache_key_for_force, force: force, next_expiry: next_expiry)
-        QaServer.config.monitor_logger.debug("(QaServer::PerformanceDailyGraphCache) - GENERATING daily performance graphs (force: #{force})")
-        generate_graphs_for_authorities
-      end
-
-      private
-
-        def generate_graphs_for_authorities
-          auths = authority_list_class.authorities_list
-          generate_graphs_for_authority(authority_name: ALL_AUTH) # generates graph for all authorities
-          auths.each { |authname| generate_graphs_for_authority(authority_name: authname) }
-        end
-
-        def generate_graphs_for_authority(authority_name:)
-          [SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
-            hash[action] = generate_30_day_graph(authority_name: authority_name, action: action)
-          end
-        end
-
-        def generate_30_day_graph(authority_name:, action:)
-          # real expiration or force caught by cache_expired?  So if we are here, either the cache has expired
-          # or force was requested.  We still expire the cache and use ttl to catch race conditions.
-          Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
-                            expires_in: next_expiry, race_condition_ttl: 1.hour, force: true) do
-            data = graph_data_service.calculate_last_30_days(authority_name: authority_name, action: action)
-            graphing_service.generate_daily_graph(authority_name: authority_name, action: action, data: data)
-            data
-          end
-        end
-
-        def cache_key_for_authority_action(authority_name:, action:)
-          "#{PERFORMANCE_GRAPH_DAILY_DATA_CACHE_KEY}--#{authority_name}--#{action}"
-        end
-
-        def cache_key_for_force
-          "#{PERFORMANCE_GRAPH_DAILY_DATA_CACHE_KEY}--force"
-        end
-
-        def next_expiry
-          QaServer::CacheExpiryService.cache_expiry
-        end
-    end
-  end
-end

data/app/cache_processors/qa_server/performance_hourly_graph_cache.rb

@@ -1,65 +0,0 @@
-# frozen_string_literal: true
-# Generate graphs for the past 24 hours using cached data.  The last hour of data is always calculated and all graphs
-# for are generated.
-module QaServer
-  class PerformanceHourlyGraphCache
-    class_attribute :authority_list_class, :graph_data_service, :graphing_service
-    self.authority_list_class = QaServer::AuthorityListerService
-    self.graph_data_service = QaServer::PerformanceGraphDataService
-    self.graphing_service = QaServer::PerformanceGraphingService
-
-    class << self
-      include QaServer::CacheKeys
-      include QaServer::PerformanceHistoryDataKeys
-
-      # Generates graphs for the past 24 hours for :search, :fetch, and :all actions for each authority.
-      # @param force [Boolean] if true, run the tests even if the cache hasn't expired; otherwise, use cache if not expired
-      def generate_graphs(force: false)
-        QaServer.config.monitor_logger.debug("(QaServer::PerformanceHourlyGraphCache) - GENERATING hourly performance graphs (force: #{force})")
-        QaServer.config.performance_cache.write_all
-        generate_graphs_for_authorities(force: force)
-      end
-
-      private
-
-        def generate_graphs_for_authorities(force:)
-          auths = authority_list_class.authorities_list
-          generate_graphs_for_authority(authority_name: ALL_AUTH, force: force) # generates graph for all authorities
-          auths.each { |authname| generate_graphs_for_authority(authority_name: authname, force: force) }
-        end
-
-        def generate_graphs_for_authority(authority_name:, force:)
-          [SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
-            hash[action] = generate_24_hour_graph(authority_name: authority_name, action: action, force: force)
-          end
-        end
-
-        def generate_24_hour_graph(authority_name:, action:, force:)
-          graph_created = false
-          data = Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
-                                   expires_in: QaServer::TimeService.current_time.end_of_hour - QaServer::TimeService.current_time,
-                                   race_condition_ttl: 1.hour, force: force) do
-            data = graph_data_service.calculate_last_24_hours(authority_name: authority_name, action: action)
-            graphing_service.generate_hourly_graph(authority_name: authority_name, action: action, data: data)
-            graph_created = true
-            data
-          end
-          regen_last_hour_and_graph(authority_name: authority_name, action: action, data: data) unless graph_created
-        end
-
-        def regen_last_hour_and_graph(authority_name:, action:, data:)
-          Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
-                            expires_in: QaServer::TimeService.current_time.end_of_hour - QaServer::TimeService.current_time,
-                            race_condition_ttl: 1.hour, force: true) do
-            data = graph_data_service.recalculate_last_hour(authority_name: authority_name, action: action, averages: data)
-            graphing_service.generate_hourly_graph(authority_name: authority_name, action: action, data: data)
-            data
-          end
-        end
-
-        def cache_key_for_authority_action(authority_name:, action:)
-          "#{PERFORMANCE_GRAPH_HOURLY_DATA_CACHE_KEY}--#{authority_name}--#{action}"
-        end
-    end
-  end
-end

data/app/cache_processors/qa_server/performance_monthly_graph_cache.rb

@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-# Generate graphs for the past 12 months using cached data.  Graphs are generated only if the cache has expired.
-module QaServer
-  class PerformanceMonthlyGraphCache
-    class_attribute :authority_list_class, :graph_data_service, :graphing_service
-    self.authority_list_class = QaServer::AuthorityListerService
-    self.graph_data_service = QaServer::PerformanceGraphDataService
-    self.graphing_service = QaServer::PerformanceGraphingService
-
-    class << self
-      include QaServer::CacheKeys
-      include QaServer::PerformanceHistoryDataKeys
-
-      # Generates graphs for the past 30 days for :search, :fetch, and :all actions for each authority.
-      # @param force [Boolean] if true, run the tests even if the cache hasn't expired; otherwise, use cache if not expired
-      def generate_graphs(force: false)
-        return unless QaServer::CacheExpiryService.cache_expired?(key: cache_key_for_force, force: force, next_expiry: next_expiry)
-        QaServer.config.monitor_logger.debug("(QaServer::PerformanceMonthlyGraphCache) - GENERATING monthly performance graphs (force: #{force})")
-        generate_graphs_for_authorities
-      end
-
-      private
-
-        def generate_graphs_for_authorities
-          auths = authority_list_class.authorities_list
-          generate_graphs_for_authority(authority_name: ALL_AUTH) # generates graph for all authorities
-          auths.each { |authname| generate_graphs_for_authority(authority_name: authname) }
-        end
-
-        def generate_graphs_for_authority(authority_name:)
-          [SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
-            hash[action] = generate_12_month_graph(authority_name: authority_name, action: action)
-          end
-        end
-
-        def generate_12_month_graph(authority_name:, action:)
-          # real expiration or force caught by cache_expired?  So if we are here, either the cache has expired
-          # or force was requested.  We still expire the cache and use ttl to catch race conditions.
-          Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
-                            expires_in: next_expiry, race_condition_ttl: 1.hour, force: true) do
-            data = graph_data_service.calculate_last_12_months(authority_name: authority_name, action: action)
-            graphing_service.generate_monthly_graph(authority_name: authority_name, action: action, data: data)
-            data
-          end
-        end
-
-        def cache_key_for_authority_action(authority_name:, action:)
-          "#{PERFORMANCE_GRAPH_MONTHLY_DATA_CACHE_KEY}--#{authority_name}--#{action}"
-        end
-
-        def cache_key_for_force
-          "#{PERFORMANCE_GRAPH_MONTHLY_DATA_CACHE_KEY}--force"
-        end
-
-        def next_expiry
-          QaServer::CacheExpiryService.cache_expiry
-        end
-    end
-  end
-end