qa_server 7.2.1 → 7.6.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (116) hide show
  1. checksums.yaml +5 -5
  2. data/.rubocop_fixme.yml +3 -0
  3. data/.travis.yml +4 -5
  4. data/CHANGELOG.md +41 -0
  5. data/Rakefile +1 -1
  6. data/app/assets/stylesheets/qa_server/_check-status.scss +36 -0
  7. data/app/cache_processors/concerns/qa_server/cache_keys.rb +0 -5
  8. data/app/cache_processors/qa_server/cache_expiry_service.rb +13 -8
  9. data/app/cache_processors/qa_server/job_id_cache.rb +29 -0
  10. data/app/cache_processors/qa_server/performance_cache.rb +34 -34
  11. data/app/cache_processors/qa_server/performance_day_graph_cache.rb +27 -0
  12. data/app/cache_processors/qa_server/performance_month_graph_cache.rb +27 -0
  13. data/app/cache_processors/qa_server/performance_year_graph_cache.rb +27 -0
  14. data/app/cache_processors/qa_server/scenario_history_cache.rb +7 -7
  15. data/app/cache_processors/qa_server/scenario_history_graph_cache.rb +12 -17
  16. data/app/cache_processors/qa_server/scenario_run_cache.rb +8 -8
  17. data/app/cache_processors/qa_server/scenario_run_failures_cache.rb +7 -7
  18. data/app/cache_processors/qa_server/scenario_run_summary_cache.rb +7 -7
  19. data/app/controllers/concerns/qa_server/authority_validation_behavior.rb +49 -44
  20. data/app/controllers/qa_server/check_status_controller.rb +92 -22
  21. data/app/controllers/qa_server/fetch_controller.rb +36 -36
  22. data/app/controllers/qa_server/monitor_status_controller.rb +108 -108
  23. data/app/jobs/qa_server/history_graph_job.rb +28 -0
  24. data/app/jobs/qa_server/monitor_tests_job.rb +19 -39
  25. data/app/jobs/qa_server/performance_day_graph_job.rb +45 -0
  26. data/app/jobs/qa_server/performance_month_graph_job.rb +45 -0
  27. data/app/jobs/qa_server/performance_per_byte_job.rb +85 -0
  28. data/app/jobs/qa_server/performance_year_graph_job.rb +45 -0
  29. data/app/loggers/qa_server/scenario_logger.rb +74 -4
  30. data/app/models/concerns/qa_server/performance_history_data_keys.rb +8 -0
  31. data/app/models/qa_server/authority_scenario.rb +4 -4
  32. data/app/models/qa_server/authority_status.rb +2 -2
  33. data/app/models/qa_server/authority_status_failure.rb +1 -1
  34. data/app/models/qa_server/performance_history.rb +2 -2
  35. data/app/models/qa_server/scenario_run_history.rb +52 -52
  36. data/app/models/qa_server/scenario_run_registry.rb +2 -2
  37. data/app/models/qa_server/scenarios.rb +26 -26
  38. data/app/models/qa_server/search_scenario.rb +24 -13
  39. data/app/models/qa_server/term_scenario.rb +29 -29
  40. data/app/prepends/prepended_linked_data/find_term.rb +40 -40
  41. data/app/prepends/prepended_linked_data/search_query.rb +36 -36
  42. data/app/prepends/prepended_rdf/rdf_graph.rb +7 -7
  43. data/app/presenters/concerns/qa_server/monitor_status/performance_datatable_behavior.rb +32 -32
  44. data/app/presenters/concerns/qa_server/monitor_status/performance_graph_behavior.rb +64 -64
  45. data/app/presenters/qa_server/check_status_presenter.rb +63 -7
  46. data/app/presenters/qa_server/monitor_status/current_status_presenter.rb +9 -8
  47. data/app/services/concerns/qa_server/gruff_graph.rb +16 -16
  48. data/app/services/qa_server/authority_loader_service.rb +14 -14
  49. data/app/services/qa_server/authority_validator_service.rb +1 -0
  50. data/app/services/qa_server/database_migrator.rb +14 -14
  51. data/app/services/qa_server/history_graphing_service.rb +30 -30
  52. data/app/services/qa_server/performance_calculator_service.rb +80 -80
  53. data/app/services/qa_server/performance_datatable_service.rb +35 -35
  54. data/app/services/qa_server/performance_graph_data_service.rb +28 -28
  55. data/app/services/qa_server/performance_graphing_service.rb +58 -58
  56. data/app/services/qa_server/performance_per_byte_calculator_service.rb +88 -0
  57. data/app/services/qa_server/performance_per_byte_data_service.rb +41 -0
  58. data/app/services/qa_server/scenarios_loader_service.rb +1 -1
  59. data/app/services/qa_server/time_period_service.rb +21 -21
  60. data/app/validators/qa_server/scenario_validator.rb +99 -87
  61. data/app/validators/qa_server/search_scenario_validator.rb +67 -61
  62. data/app/validators/qa_server/term_scenario_validator.rb +20 -15
  63. data/app/views/qa_server/check_status/index.html.erb +120 -24
  64. data/app/views/qa_server/monitor_status/_test_summary.html.erb +1 -1
  65. data/config/i18n-tasks.yml +133 -0
  66. data/config/locales/qa_server.en.yml +16 -0
  67. data/lib/generators/qa_server/assets_generator.rb +4 -4
  68. data/lib/generators/qa_server/templates/config/authorities/linked_data/cerl_ld4l_cache.json +2 -2
  69. data/lib/generators/qa_server/templates/config/authorities/linked_data/getty_aat_ld4l_cache.json +62 -1
  70. data/lib/generators/qa_server/templates/config/authorities/linked_data/getty_tgn_ld4l_cache.json +45 -11
  71. data/lib/generators/qa_server/templates/config/authorities/linked_data/getty_ulan_ld4l_cache.json +104 -8
  72. data/lib/generators/qa_server/templates/config/authorities/linked_data/isni_ld4l_cache.json +90 -0
  73. data/lib/generators/qa_server/templates/config/authorities/linked_data/ligatus_ld4l_cache.json +133 -0
  74. data/lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo2_ld4l_cache.json +248 -0
  75. data/lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo3_ld4l_cache.json +248 -0
  76. data/lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo_ld4l_cache.json +4 -4
  77. data/lib/generators/qa_server/templates/config/authorities/linked_data/locvocabs_ld4l_cache.json +117 -0
  78. data/lib/generators/qa_server/templates/config/authorities/linked_data/mesh_nlm_ld4l_cache.json +135 -3
  79. data/lib/generators/qa_server/templates/config/authorities/linked_data/oclcfast_direct.json +5 -0
  80. data/lib/generators/qa_server/templates/config/authorities/linked_data/oclcfast_ld4l_cache.json +2 -4
  81. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/agrovoc_direct_validation.yml +31 -0
  82. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/agrovoc_ld4l_cache_validation.yml +31 -0
  83. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/cerl_ld4l_cache_validation.yml +26 -14
  84. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/dbpedia_ld4l_cache_validation.yml +33 -0
  85. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/geonames_direct_validation.yml +35 -0
  86. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/geonames_ld4l_cache_validation.yml +55 -5
  87. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/getty_aat_ld4l_cache_validation.yml +253 -0
  88. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/getty_tgn_ld4l_cache_validation.yml +31 -1
  89. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/getty_ulan_ld4l_cache_validation.yml +38 -1
  90. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/isni_ld4l_cache_validation.yml +10 -0
  91. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/ligatus_ld4l_cache_validation.yml +36 -0
  92. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locdemographics_ld4l_cache_validation.yml +69 -44
  93. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locgenres_ld4l_cache_validation.yml +22 -0
  94. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_ld4l_cache_validation.yml +65 -0
  95. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_rwo2_ld4l_cache_validation.yml +78 -0
  96. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_rwo3_ld4l_cache_validation.yml +73 -0
  97. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_rwo_ld4l_cache_validation.yml +71 -3
  98. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locperformance_ld4l_cache_validation.yml +6 -0
  99. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locsubjects_ld4l_cache_validation.yml +30 -0
  100. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locvocabs_ld4l_cache_validation.yml +430 -0
  101. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/mesh_nlm_ld4l_cache_validation.yml +54 -1
  102. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/nalt_ld4l_cache_validation.yml +37 -0
  103. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/oclc_fast_validation.yml +71 -5
  104. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/oclcfast_direct_validation.yml +73 -2
  105. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/oclcfast_ld4l_cache_validation.yml +73 -0
  106. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/rda_registry_ld4l_cache_validation.yml +307 -0
  107. data/lib/qa_server/configuration.rb +28 -24
  108. data/lib/qa_server/version.rb +1 -1
  109. data/qa_server.gemspec +6 -5
  110. data/spec/feature/accuracy_spec.rb +32 -0
  111. data/spec/i18n_spec.rb +36 -0
  112. data/spec/spec_helper.rb +4 -0
  113. metadata +74 -20
  114. data/app/cache_processors/qa_server/performance_daily_graph_cache.rb +0 -60
  115. data/app/cache_processors/qa_server/performance_hourly_graph_cache.rb +0 -65
  116. data/app/cache_processors/qa_server/performance_monthly_graph_cache.rb +0 -60
@@ -1,60 +0,0 @@
1
- # frozen_string_literal: true
2
- # Generate graphs for the past 30 days using cached data. Graphs are generated only if the cache has expired.
3
- module QaServer
4
- class PerformanceDailyGraphCache
5
- class_attribute :authority_list_class, :graph_data_service, :graphing_service
6
- self.authority_list_class = QaServer::AuthorityListerService
7
- self.graph_data_service = QaServer::PerformanceGraphDataService
8
- self.graphing_service = QaServer::PerformanceGraphingService
9
-
10
- class << self
11
- include QaServer::CacheKeys
12
- include QaServer::PerformanceHistoryDataKeys
13
-
14
- # Generates graphs for the past 30 days for :search, :fetch, and :all actions for each authority.
15
- # @param force [Boolean] if true, run the tests even if the cache hasn't expired; otherwise, use cache if not expired
16
- def generate_graphs(force: false)
17
- return unless QaServer::CacheExpiryService.cache_expired?(key: cache_key_for_force, force: force, next_expiry: next_expiry)
18
- QaServer.config.monitor_logger.debug("(QaServer::PerformanceDailyGraphCache) - GENERATING daily performance graphs (force: #{force})")
19
- generate_graphs_for_authorities
20
- end
21
-
22
- private
23
-
24
- def generate_graphs_for_authorities
25
- auths = authority_list_class.authorities_list
26
- generate_graphs_for_authority(authority_name: ALL_AUTH) # generates graph for all authorities
27
- auths.each { |authname| generate_graphs_for_authority(authority_name: authname) }
28
- end
29
-
30
- def generate_graphs_for_authority(authority_name:)
31
- [SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
32
- hash[action] = generate_30_day_graph(authority_name: authority_name, action: action)
33
- end
34
- end
35
-
36
- def generate_30_day_graph(authority_name:, action:)
37
- # real expiration or force caught by cache_expired? So if we are here, either the cache has expired
38
- # or force was requested. We still expire the cache and use ttl to catch race conditions.
39
- Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
40
- expires_in: next_expiry, race_condition_ttl: 1.hour, force: true) do
41
- data = graph_data_service.calculate_last_30_days(authority_name: authority_name, action: action)
42
- graphing_service.generate_daily_graph(authority_name: authority_name, action: action, data: data)
43
- data
44
- end
45
- end
46
-
47
- def cache_key_for_authority_action(authority_name:, action:)
48
- "#{PERFORMANCE_GRAPH_DAILY_DATA_CACHE_KEY}--#{authority_name}--#{action}"
49
- end
50
-
51
- def cache_key_for_force
52
- "#{PERFORMANCE_GRAPH_DAILY_DATA_CACHE_KEY}--force"
53
- end
54
-
55
- def next_expiry
56
- QaServer::CacheExpiryService.cache_expiry
57
- end
58
- end
59
- end
60
- end
@@ -1,65 +0,0 @@
1
- # frozen_string_literal: true
2
- # Generate graphs for the past 24 hours using cached data. The last hour of data is always calculated and all graphs
3
- # for are generated.
4
- module QaServer
5
- class PerformanceHourlyGraphCache
6
- class_attribute :authority_list_class, :graph_data_service, :graphing_service
7
- self.authority_list_class = QaServer::AuthorityListerService
8
- self.graph_data_service = QaServer::PerformanceGraphDataService
9
- self.graphing_service = QaServer::PerformanceGraphingService
10
-
11
- class << self
12
- include QaServer::CacheKeys
13
- include QaServer::PerformanceHistoryDataKeys
14
-
15
- # Generates graphs for the past 24 hours for :search, :fetch, and :all actions for each authority.
16
- # @param force [Boolean] if true, run the tests even if the cache hasn't expired; otherwise, use cache if not expired
17
- def generate_graphs(force: false)
18
- QaServer.config.monitor_logger.debug("(QaServer::PerformanceHourlyGraphCache) - GENERATING hourly performance graphs (force: #{force})")
19
- QaServer.config.performance_cache.write_all
20
- generate_graphs_for_authorities(force: force)
21
- end
22
-
23
- private
24
-
25
- def generate_graphs_for_authorities(force:)
26
- auths = authority_list_class.authorities_list
27
- generate_graphs_for_authority(authority_name: ALL_AUTH, force: force) # generates graph for all authorities
28
- auths.each { |authname| generate_graphs_for_authority(authority_name: authname, force: force) }
29
- end
30
-
31
- def generate_graphs_for_authority(authority_name:, force:)
32
- [SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
33
- hash[action] = generate_24_hour_graph(authority_name: authority_name, action: action, force: force)
34
- end
35
- end
36
-
37
- def generate_24_hour_graph(authority_name:, action:, force:)
38
- graph_created = false
39
- data = Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
40
- expires_in: QaServer::TimeService.current_time.end_of_hour - QaServer::TimeService.current_time,
41
- race_condition_ttl: 1.hour, force: force) do
42
- data = graph_data_service.calculate_last_24_hours(authority_name: authority_name, action: action)
43
- graphing_service.generate_hourly_graph(authority_name: authority_name, action: action, data: data)
44
- graph_created = true
45
- data
46
- end
47
- regen_last_hour_and_graph(authority_name: authority_name, action: action, data: data) unless graph_created
48
- end
49
-
50
- def regen_last_hour_and_graph(authority_name:, action:, data:)
51
- Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
52
- expires_in: QaServer::TimeService.current_time.end_of_hour - QaServer::TimeService.current_time,
53
- race_condition_ttl: 1.hour, force: true) do
54
- data = graph_data_service.recalculate_last_hour(authority_name: authority_name, action: action, averages: data)
55
- graphing_service.generate_hourly_graph(authority_name: authority_name, action: action, data: data)
56
- data
57
- end
58
- end
59
-
60
- def cache_key_for_authority_action(authority_name:, action:)
61
- "#{PERFORMANCE_GRAPH_HOURLY_DATA_CACHE_KEY}--#{authority_name}--#{action}"
62
- end
63
- end
64
- end
65
- end
@@ -1,60 +0,0 @@
1
- # frozen_string_literal: true
2
- # Generate graphs for the past 12 months using cached data. Graphs are generated only if the cache has expired.
3
- module QaServer
4
- class PerformanceMonthlyGraphCache
5
- class_attribute :authority_list_class, :graph_data_service, :graphing_service
6
- self.authority_list_class = QaServer::AuthorityListerService
7
- self.graph_data_service = QaServer::PerformanceGraphDataService
8
- self.graphing_service = QaServer::PerformanceGraphingService
9
-
10
- class << self
11
- include QaServer::CacheKeys
12
- include QaServer::PerformanceHistoryDataKeys
13
-
14
- # Generates graphs for the past 30 days for :search, :fetch, and :all actions for each authority.
15
- # @param force [Boolean] if true, run the tests even if the cache hasn't expired; otherwise, use cache if not expired
16
- def generate_graphs(force: false)
17
- return unless QaServer::CacheExpiryService.cache_expired?(key: cache_key_for_force, force: force, next_expiry: next_expiry)
18
- QaServer.config.monitor_logger.debug("(QaServer::PerformanceMonthlyGraphCache) - GENERATING monthly performance graphs (force: #{force})")
19
- generate_graphs_for_authorities
20
- end
21
-
22
- private
23
-
24
- def generate_graphs_for_authorities
25
- auths = authority_list_class.authorities_list
26
- generate_graphs_for_authority(authority_name: ALL_AUTH) # generates graph for all authorities
27
- auths.each { |authname| generate_graphs_for_authority(authority_name: authname) }
28
- end
29
-
30
- def generate_graphs_for_authority(authority_name:)
31
- [SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
32
- hash[action] = generate_12_month_graph(authority_name: authority_name, action: action)
33
- end
34
- end
35
-
36
- def generate_12_month_graph(authority_name:, action:)
37
- # real expiration or force caught by cache_expired? So if we are here, either the cache has expired
38
- # or force was requested. We still expire the cache and use ttl to catch race conditions.
39
- Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
40
- expires_in: next_expiry, race_condition_ttl: 1.hour, force: true) do
41
- data = graph_data_service.calculate_last_12_months(authority_name: authority_name, action: action)
42
- graphing_service.generate_monthly_graph(authority_name: authority_name, action: action, data: data)
43
- data
44
- end
45
- end
46
-
47
- def cache_key_for_authority_action(authority_name:, action:)
48
- "#{PERFORMANCE_GRAPH_MONTHLY_DATA_CACHE_KEY}--#{authority_name}--#{action}"
49
- end
50
-
51
- def cache_key_for_force
52
- "#{PERFORMANCE_GRAPH_MONTHLY_DATA_CACHE_KEY}--force"
53
- end
54
-
55
- def next_expiry
56
- QaServer::CacheExpiryService.cache_expiry
57
- end
58
- end
59
- end
60
- end