qa_server 7.2.1 → 7.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. checksums.yaml +5 -5
  2. data/.rubocop_fixme.yml +3 -0
  3. data/.travis.yml +4 -5
  4. data/CHANGELOG.md +41 -0
  5. data/Rakefile +1 -1
  6. data/app/assets/stylesheets/qa_server/_check-status.scss +36 -0
  7. data/app/cache_processors/concerns/qa_server/cache_keys.rb +0 -5
  8. data/app/cache_processors/qa_server/cache_expiry_service.rb +13 -8
  9. data/app/cache_processors/qa_server/job_id_cache.rb +29 -0
  10. data/app/cache_processors/qa_server/performance_cache.rb +34 -34
  11. data/app/cache_processors/qa_server/performance_day_graph_cache.rb +27 -0
  12. data/app/cache_processors/qa_server/performance_month_graph_cache.rb +27 -0
  13. data/app/cache_processors/qa_server/performance_year_graph_cache.rb +27 -0
  14. data/app/cache_processors/qa_server/scenario_history_cache.rb +7 -7
  15. data/app/cache_processors/qa_server/scenario_history_graph_cache.rb +12 -17
  16. data/app/cache_processors/qa_server/scenario_run_cache.rb +8 -8
  17. data/app/cache_processors/qa_server/scenario_run_failures_cache.rb +7 -7
  18. data/app/cache_processors/qa_server/scenario_run_summary_cache.rb +7 -7
  19. data/app/controllers/concerns/qa_server/authority_validation_behavior.rb +49 -44
  20. data/app/controllers/qa_server/check_status_controller.rb +92 -22
  21. data/app/controllers/qa_server/fetch_controller.rb +36 -36
  22. data/app/controllers/qa_server/monitor_status_controller.rb +108 -108
  23. data/app/jobs/qa_server/history_graph_job.rb +28 -0
  24. data/app/jobs/qa_server/monitor_tests_job.rb +19 -39
  25. data/app/jobs/qa_server/performance_day_graph_job.rb +45 -0
  26. data/app/jobs/qa_server/performance_month_graph_job.rb +45 -0
  27. data/app/jobs/qa_server/performance_per_byte_job.rb +85 -0
  28. data/app/jobs/qa_server/performance_year_graph_job.rb +45 -0
  29. data/app/loggers/qa_server/scenario_logger.rb +74 -4
  30. data/app/models/concerns/qa_server/performance_history_data_keys.rb +8 -0
  31. data/app/models/qa_server/authority_scenario.rb +4 -4
  32. data/app/models/qa_server/authority_status.rb +2 -2
  33. data/app/models/qa_server/authority_status_failure.rb +1 -1
  34. data/app/models/qa_server/performance_history.rb +2 -2
  35. data/app/models/qa_server/scenario_run_history.rb +52 -52
  36. data/app/models/qa_server/scenario_run_registry.rb +2 -2
  37. data/app/models/qa_server/scenarios.rb +26 -26
  38. data/app/models/qa_server/search_scenario.rb +24 -13
  39. data/app/models/qa_server/term_scenario.rb +29 -29
  40. data/app/prepends/prepended_linked_data/find_term.rb +40 -40
  41. data/app/prepends/prepended_linked_data/search_query.rb +36 -36
  42. data/app/prepends/prepended_rdf/rdf_graph.rb +7 -7
  43. data/app/presenters/concerns/qa_server/monitor_status/performance_datatable_behavior.rb +32 -32
  44. data/app/presenters/concerns/qa_server/monitor_status/performance_graph_behavior.rb +64 -64
  45. data/app/presenters/qa_server/check_status_presenter.rb +63 -7
  46. data/app/presenters/qa_server/monitor_status/current_status_presenter.rb +9 -8
  47. data/app/services/concerns/qa_server/gruff_graph.rb +16 -16
  48. data/app/services/qa_server/authority_loader_service.rb +14 -14
  49. data/app/services/qa_server/authority_validator_service.rb +1 -0
  50. data/app/services/qa_server/database_migrator.rb +14 -14
  51. data/app/services/qa_server/history_graphing_service.rb +30 -30
  52. data/app/services/qa_server/performance_calculator_service.rb +80 -80
  53. data/app/services/qa_server/performance_datatable_service.rb +35 -35
  54. data/app/services/qa_server/performance_graph_data_service.rb +28 -28
  55. data/app/services/qa_server/performance_graphing_service.rb +58 -58
  56. data/app/services/qa_server/performance_per_byte_calculator_service.rb +88 -0
  57. data/app/services/qa_server/performance_per_byte_data_service.rb +41 -0
  58. data/app/services/qa_server/scenarios_loader_service.rb +1 -1
  59. data/app/services/qa_server/time_period_service.rb +21 -21
  60. data/app/validators/qa_server/scenario_validator.rb +99 -87
  61. data/app/validators/qa_server/search_scenario_validator.rb +67 -61
  62. data/app/validators/qa_server/term_scenario_validator.rb +20 -15
  63. data/app/views/qa_server/check_status/index.html.erb +120 -24
  64. data/app/views/qa_server/monitor_status/_test_summary.html.erb +1 -1
  65. data/config/i18n-tasks.yml +133 -0
  66. data/config/locales/qa_server.en.yml +16 -0
  67. data/lib/generators/qa_server/assets_generator.rb +4 -4
  68. data/lib/generators/qa_server/templates/config/authorities/linked_data/cerl_ld4l_cache.json +2 -2
  69. data/lib/generators/qa_server/templates/config/authorities/linked_data/getty_aat_ld4l_cache.json +62 -1
  70. data/lib/generators/qa_server/templates/config/authorities/linked_data/getty_tgn_ld4l_cache.json +45 -11
  71. data/lib/generators/qa_server/templates/config/authorities/linked_data/getty_ulan_ld4l_cache.json +104 -8
  72. data/lib/generators/qa_server/templates/config/authorities/linked_data/isni_ld4l_cache.json +90 -0
  73. data/lib/generators/qa_server/templates/config/authorities/linked_data/ligatus_ld4l_cache.json +133 -0
  74. data/lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo2_ld4l_cache.json +248 -0
  75. data/lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo3_ld4l_cache.json +248 -0
  76. data/lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo_ld4l_cache.json +4 -4
  77. data/lib/generators/qa_server/templates/config/authorities/linked_data/locvocabs_ld4l_cache.json +117 -0
  78. data/lib/generators/qa_server/templates/config/authorities/linked_data/mesh_nlm_ld4l_cache.json +135 -3
  79. data/lib/generators/qa_server/templates/config/authorities/linked_data/oclcfast_direct.json +5 -0
  80. data/lib/generators/qa_server/templates/config/authorities/linked_data/oclcfast_ld4l_cache.json +2 -4
  81. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/agrovoc_direct_validation.yml +31 -0
  82. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/agrovoc_ld4l_cache_validation.yml +31 -0
  83. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/cerl_ld4l_cache_validation.yml +26 -14
  84. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/dbpedia_ld4l_cache_validation.yml +33 -0
  85. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/geonames_direct_validation.yml +35 -0
  86. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/geonames_ld4l_cache_validation.yml +55 -5
  87. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/getty_aat_ld4l_cache_validation.yml +253 -0
  88. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/getty_tgn_ld4l_cache_validation.yml +31 -1
  89. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/getty_ulan_ld4l_cache_validation.yml +38 -1
  90. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/isni_ld4l_cache_validation.yml +10 -0
  91. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/ligatus_ld4l_cache_validation.yml +36 -0
  92. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locdemographics_ld4l_cache_validation.yml +69 -44
  93. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locgenres_ld4l_cache_validation.yml +22 -0
  94. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_ld4l_cache_validation.yml +65 -0
  95. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_rwo2_ld4l_cache_validation.yml +78 -0
  96. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_rwo3_ld4l_cache_validation.yml +73 -0
  97. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_rwo_ld4l_cache_validation.yml +71 -3
  98. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locperformance_ld4l_cache_validation.yml +6 -0
  99. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locsubjects_ld4l_cache_validation.yml +30 -0
  100. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locvocabs_ld4l_cache_validation.yml +430 -0
  101. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/mesh_nlm_ld4l_cache_validation.yml +54 -1
  102. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/nalt_ld4l_cache_validation.yml +37 -0
  103. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/oclc_fast_validation.yml +71 -5
  104. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/oclcfast_direct_validation.yml +73 -2
  105. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/oclcfast_ld4l_cache_validation.yml +73 -0
  106. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/rda_registry_ld4l_cache_validation.yml +307 -0
  107. data/lib/qa_server/configuration.rb +28 -24
  108. data/lib/qa_server/version.rb +1 -1
  109. data/qa_server.gemspec +6 -5
  110. data/spec/feature/accuracy_spec.rb +32 -0
  111. data/spec/i18n_spec.rb +36 -0
  112. data/spec/spec_helper.rb +4 -0
  113. metadata +74 -20
  114. data/app/cache_processors/qa_server/performance_daily_graph_cache.rb +0 -60
  115. data/app/cache_processors/qa_server/performance_hourly_graph_cache.rb +0 -65
  116. data/app/cache_processors/qa_server/performance_monthly_graph_cache.rb +0 -60
@@ -1,60 +0,0 @@
1
- # frozen_string_literal: true
2
- # Generate graphs for the past 30 days using cached data. Graphs are generated only if the cache has expired.
3
- module QaServer
4
- class PerformanceDailyGraphCache
5
- class_attribute :authority_list_class, :graph_data_service, :graphing_service
6
- self.authority_list_class = QaServer::AuthorityListerService
7
- self.graph_data_service = QaServer::PerformanceGraphDataService
8
- self.graphing_service = QaServer::PerformanceGraphingService
9
-
10
- class << self
11
- include QaServer::CacheKeys
12
- include QaServer::PerformanceHistoryDataKeys
13
-
14
- # Generates graphs for the past 30 days for :search, :fetch, and :all actions for each authority.
15
- # @param force [Boolean] if true, run the tests even if the cache hasn't expired; otherwise, use cache if not expired
16
- def generate_graphs(force: false)
17
- return unless QaServer::CacheExpiryService.cache_expired?(key: cache_key_for_force, force: force, next_expiry: next_expiry)
18
- QaServer.config.monitor_logger.debug("(QaServer::PerformanceDailyGraphCache) - GENERATING daily performance graphs (force: #{force})")
19
- generate_graphs_for_authorities
20
- end
21
-
22
- private
23
-
24
- def generate_graphs_for_authorities
25
- auths = authority_list_class.authorities_list
26
- generate_graphs_for_authority(authority_name: ALL_AUTH) # generates graph for all authorities
27
- auths.each { |authname| generate_graphs_for_authority(authority_name: authname) }
28
- end
29
-
30
- def generate_graphs_for_authority(authority_name:)
31
- [SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
32
- hash[action] = generate_30_day_graph(authority_name: authority_name, action: action)
33
- end
34
- end
35
-
36
- def generate_30_day_graph(authority_name:, action:)
37
- # real expiration or force caught by cache_expired? So if we are here, either the cache has expired
38
- # or force was requested. We still expire the cache and use ttl to catch race conditions.
39
- Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
40
- expires_in: next_expiry, race_condition_ttl: 1.hour, force: true) do
41
- data = graph_data_service.calculate_last_30_days(authority_name: authority_name, action: action)
42
- graphing_service.generate_daily_graph(authority_name: authority_name, action: action, data: data)
43
- data
44
- end
45
- end
46
-
47
- def cache_key_for_authority_action(authority_name:, action:)
48
- "#{PERFORMANCE_GRAPH_DAILY_DATA_CACHE_KEY}--#{authority_name}--#{action}"
49
- end
50
-
51
- def cache_key_for_force
52
- "#{PERFORMANCE_GRAPH_DAILY_DATA_CACHE_KEY}--force"
53
- end
54
-
55
- def next_expiry
56
- QaServer::CacheExpiryService.cache_expiry
57
- end
58
- end
59
- end
60
- end
@@ -1,65 +0,0 @@
1
- # frozen_string_literal: true
2
- # Generate graphs for the past 24 hours using cached data. The last hour of data is always calculated and all graphs
3
- # for are generated.
4
- module QaServer
5
- class PerformanceHourlyGraphCache
6
- class_attribute :authority_list_class, :graph_data_service, :graphing_service
7
- self.authority_list_class = QaServer::AuthorityListerService
8
- self.graph_data_service = QaServer::PerformanceGraphDataService
9
- self.graphing_service = QaServer::PerformanceGraphingService
10
-
11
- class << self
12
- include QaServer::CacheKeys
13
- include QaServer::PerformanceHistoryDataKeys
14
-
15
- # Generates graphs for the past 24 hours for :search, :fetch, and :all actions for each authority.
16
- # @param force [Boolean] if true, run the tests even if the cache hasn't expired; otherwise, use cache if not expired
17
- def generate_graphs(force: false)
18
- QaServer.config.monitor_logger.debug("(QaServer::PerformanceHourlyGraphCache) - GENERATING hourly performance graphs (force: #{force})")
19
- QaServer.config.performance_cache.write_all
20
- generate_graphs_for_authorities(force: force)
21
- end
22
-
23
- private
24
-
25
- def generate_graphs_for_authorities(force:)
26
- auths = authority_list_class.authorities_list
27
- generate_graphs_for_authority(authority_name: ALL_AUTH, force: force) # generates graph for all authorities
28
- auths.each { |authname| generate_graphs_for_authority(authority_name: authname, force: force) }
29
- end
30
-
31
- def generate_graphs_for_authority(authority_name:, force:)
32
- [SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
33
- hash[action] = generate_24_hour_graph(authority_name: authority_name, action: action, force: force)
34
- end
35
- end
36
-
37
- def generate_24_hour_graph(authority_name:, action:, force:)
38
- graph_created = false
39
- data = Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
40
- expires_in: QaServer::TimeService.current_time.end_of_hour - QaServer::TimeService.current_time,
41
- race_condition_ttl: 1.hour, force: force) do
42
- data = graph_data_service.calculate_last_24_hours(authority_name: authority_name, action: action)
43
- graphing_service.generate_hourly_graph(authority_name: authority_name, action: action, data: data)
44
- graph_created = true
45
- data
46
- end
47
- regen_last_hour_and_graph(authority_name: authority_name, action: action, data: data) unless graph_created
48
- end
49
-
50
- def regen_last_hour_and_graph(authority_name:, action:, data:)
51
- Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
52
- expires_in: QaServer::TimeService.current_time.end_of_hour - QaServer::TimeService.current_time,
53
- race_condition_ttl: 1.hour, force: true) do
54
- data = graph_data_service.recalculate_last_hour(authority_name: authority_name, action: action, averages: data)
55
- graphing_service.generate_hourly_graph(authority_name: authority_name, action: action, data: data)
56
- data
57
- end
58
- end
59
-
60
- def cache_key_for_authority_action(authority_name:, action:)
61
- "#{PERFORMANCE_GRAPH_HOURLY_DATA_CACHE_KEY}--#{authority_name}--#{action}"
62
- end
63
- end
64
- end
65
- end
@@ -1,60 +0,0 @@
1
- # frozen_string_literal: true
2
- # Generate graphs for the past 12 months using cached data. Graphs are generated only if the cache has expired.
3
- module QaServer
4
- class PerformanceMonthlyGraphCache
5
- class_attribute :authority_list_class, :graph_data_service, :graphing_service
6
- self.authority_list_class = QaServer::AuthorityListerService
7
- self.graph_data_service = QaServer::PerformanceGraphDataService
8
- self.graphing_service = QaServer::PerformanceGraphingService
9
-
10
- class << self
11
- include QaServer::CacheKeys
12
- include QaServer::PerformanceHistoryDataKeys
13
-
14
- # Generates graphs for the past 30 days for :search, :fetch, and :all actions for each authority.
15
- # @param force [Boolean] if true, run the tests even if the cache hasn't expired; otherwise, use cache if not expired
16
- def generate_graphs(force: false)
17
- return unless QaServer::CacheExpiryService.cache_expired?(key: cache_key_for_force, force: force, next_expiry: next_expiry)
18
- QaServer.config.monitor_logger.debug("(QaServer::PerformanceMonthlyGraphCache) - GENERATING monthly performance graphs (force: #{force})")
19
- generate_graphs_for_authorities
20
- end
21
-
22
- private
23
-
24
- def generate_graphs_for_authorities
25
- auths = authority_list_class.authorities_list
26
- generate_graphs_for_authority(authority_name: ALL_AUTH) # generates graph for all authorities
27
- auths.each { |authname| generate_graphs_for_authority(authority_name: authname) }
28
- end
29
-
30
- def generate_graphs_for_authority(authority_name:)
31
- [SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
32
- hash[action] = generate_12_month_graph(authority_name: authority_name, action: action)
33
- end
34
- end
35
-
36
- def generate_12_month_graph(authority_name:, action:)
37
- # real expiration or force caught by cache_expired? So if we are here, either the cache has expired
38
- # or force was requested. We still expire the cache and use ttl to catch race conditions.
39
- Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
40
- expires_in: next_expiry, race_condition_ttl: 1.hour, force: true) do
41
- data = graph_data_service.calculate_last_12_months(authority_name: authority_name, action: action)
42
- graphing_service.generate_monthly_graph(authority_name: authority_name, action: action, data: data)
43
- data
44
- end
45
- end
46
-
47
- def cache_key_for_authority_action(authority_name:, action:)
48
- "#{PERFORMANCE_GRAPH_MONTHLY_DATA_CACHE_KEY}--#{authority_name}--#{action}"
49
- end
50
-
51
- def cache_key_for_force
52
- "#{PERFORMANCE_GRAPH_MONTHLY_DATA_CACHE_KEY}--force"
53
- end
54
-
55
- def next_expiry
56
- QaServer::CacheExpiryService.cache_expiry
57
- end
58
- end
59
- end
60
- end