qa_server 7.2.1 → 7.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.rubocop_fixme.yml +3 -0
- data/.travis.yml +4 -5
- data/CHANGELOG.md +41 -0
- data/Rakefile +1 -1
- data/app/assets/stylesheets/qa_server/_check-status.scss +36 -0
- data/app/cache_processors/concerns/qa_server/cache_keys.rb +0 -5
- data/app/cache_processors/qa_server/cache_expiry_service.rb +13 -8
- data/app/cache_processors/qa_server/job_id_cache.rb +29 -0
- data/app/cache_processors/qa_server/performance_cache.rb +34 -34
- data/app/cache_processors/qa_server/performance_day_graph_cache.rb +27 -0
- data/app/cache_processors/qa_server/performance_month_graph_cache.rb +27 -0
- data/app/cache_processors/qa_server/performance_year_graph_cache.rb +27 -0
- data/app/cache_processors/qa_server/scenario_history_cache.rb +7 -7
- data/app/cache_processors/qa_server/scenario_history_graph_cache.rb +12 -17
- data/app/cache_processors/qa_server/scenario_run_cache.rb +8 -8
- data/app/cache_processors/qa_server/scenario_run_failures_cache.rb +7 -7
- data/app/cache_processors/qa_server/scenario_run_summary_cache.rb +7 -7
- data/app/controllers/concerns/qa_server/authority_validation_behavior.rb +49 -44
- data/app/controllers/qa_server/check_status_controller.rb +92 -22
- data/app/controllers/qa_server/fetch_controller.rb +36 -36
- data/app/controllers/qa_server/monitor_status_controller.rb +108 -108
- data/app/jobs/qa_server/history_graph_job.rb +28 -0
- data/app/jobs/qa_server/monitor_tests_job.rb +19 -39
- data/app/jobs/qa_server/performance_day_graph_job.rb +45 -0
- data/app/jobs/qa_server/performance_month_graph_job.rb +45 -0
- data/app/jobs/qa_server/performance_per_byte_job.rb +85 -0
- data/app/jobs/qa_server/performance_year_graph_job.rb +45 -0
- data/app/loggers/qa_server/scenario_logger.rb +74 -4
- data/app/models/concerns/qa_server/performance_history_data_keys.rb +8 -0
- data/app/models/qa_server/authority_scenario.rb +4 -4
- data/app/models/qa_server/authority_status.rb +2 -2
- data/app/models/qa_server/authority_status_failure.rb +1 -1
- data/app/models/qa_server/performance_history.rb +2 -2
- data/app/models/qa_server/scenario_run_history.rb +52 -52
- data/app/models/qa_server/scenario_run_registry.rb +2 -2
- data/app/models/qa_server/scenarios.rb +26 -26
- data/app/models/qa_server/search_scenario.rb +24 -13
- data/app/models/qa_server/term_scenario.rb +29 -29
- data/app/prepends/prepended_linked_data/find_term.rb +40 -40
- data/app/prepends/prepended_linked_data/search_query.rb +36 -36
- data/app/prepends/prepended_rdf/rdf_graph.rb +7 -7
- data/app/presenters/concerns/qa_server/monitor_status/performance_datatable_behavior.rb +32 -32
- data/app/presenters/concerns/qa_server/monitor_status/performance_graph_behavior.rb +64 -64
- data/app/presenters/qa_server/check_status_presenter.rb +63 -7
- data/app/presenters/qa_server/monitor_status/current_status_presenter.rb +9 -8
- data/app/services/concerns/qa_server/gruff_graph.rb +16 -16
- data/app/services/qa_server/authority_loader_service.rb +14 -14
- data/app/services/qa_server/authority_validator_service.rb +1 -0
- data/app/services/qa_server/database_migrator.rb +14 -14
- data/app/services/qa_server/history_graphing_service.rb +30 -30
- data/app/services/qa_server/performance_calculator_service.rb +80 -80
- data/app/services/qa_server/performance_datatable_service.rb +35 -35
- data/app/services/qa_server/performance_graph_data_service.rb +28 -28
- data/app/services/qa_server/performance_graphing_service.rb +58 -58
- data/app/services/qa_server/performance_per_byte_calculator_service.rb +88 -0
- data/app/services/qa_server/performance_per_byte_data_service.rb +41 -0
- data/app/services/qa_server/scenarios_loader_service.rb +1 -1
- data/app/services/qa_server/time_period_service.rb +21 -21
- data/app/validators/qa_server/scenario_validator.rb +99 -87
- data/app/validators/qa_server/search_scenario_validator.rb +67 -61
- data/app/validators/qa_server/term_scenario_validator.rb +20 -15
- data/app/views/qa_server/check_status/index.html.erb +120 -24
- data/app/views/qa_server/monitor_status/_test_summary.html.erb +1 -1
- data/config/i18n-tasks.yml +133 -0
- data/config/locales/qa_server.en.yml +16 -0
- data/lib/generators/qa_server/assets_generator.rb +4 -4
- data/lib/generators/qa_server/templates/config/authorities/linked_data/cerl_ld4l_cache.json +2 -2
- data/lib/generators/qa_server/templates/config/authorities/linked_data/getty_aat_ld4l_cache.json +62 -1
- data/lib/generators/qa_server/templates/config/authorities/linked_data/getty_tgn_ld4l_cache.json +45 -11
- data/lib/generators/qa_server/templates/config/authorities/linked_data/getty_ulan_ld4l_cache.json +104 -8
- data/lib/generators/qa_server/templates/config/authorities/linked_data/isni_ld4l_cache.json +90 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/ligatus_ld4l_cache.json +133 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo2_ld4l_cache.json +248 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo3_ld4l_cache.json +248 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo_ld4l_cache.json +4 -4
- data/lib/generators/qa_server/templates/config/authorities/linked_data/locvocabs_ld4l_cache.json +117 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/mesh_nlm_ld4l_cache.json +135 -3
- data/lib/generators/qa_server/templates/config/authorities/linked_data/oclcfast_direct.json +5 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/oclcfast_ld4l_cache.json +2 -4
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/agrovoc_direct_validation.yml +31 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/agrovoc_ld4l_cache_validation.yml +31 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/cerl_ld4l_cache_validation.yml +26 -14
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/dbpedia_ld4l_cache_validation.yml +33 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/geonames_direct_validation.yml +35 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/geonames_ld4l_cache_validation.yml +55 -5
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/getty_aat_ld4l_cache_validation.yml +253 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/getty_tgn_ld4l_cache_validation.yml +31 -1
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/getty_ulan_ld4l_cache_validation.yml +38 -1
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/isni_ld4l_cache_validation.yml +10 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/ligatus_ld4l_cache_validation.yml +36 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locdemographics_ld4l_cache_validation.yml +69 -44
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locgenres_ld4l_cache_validation.yml +22 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_ld4l_cache_validation.yml +65 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_rwo2_ld4l_cache_validation.yml +78 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_rwo3_ld4l_cache_validation.yml +73 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_rwo_ld4l_cache_validation.yml +71 -3
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locperformance_ld4l_cache_validation.yml +6 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locsubjects_ld4l_cache_validation.yml +30 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locvocabs_ld4l_cache_validation.yml +430 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/mesh_nlm_ld4l_cache_validation.yml +54 -1
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/nalt_ld4l_cache_validation.yml +37 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/oclc_fast_validation.yml +71 -5
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/oclcfast_direct_validation.yml +73 -2
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/oclcfast_ld4l_cache_validation.yml +73 -0
- data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/rda_registry_ld4l_cache_validation.yml +307 -0
- data/lib/qa_server/configuration.rb +28 -24
- data/lib/qa_server/version.rb +1 -1
- data/qa_server.gemspec +6 -5
- data/spec/feature/accuracy_spec.rb +32 -0
- data/spec/i18n_spec.rb +36 -0
- data/spec/spec_helper.rb +4 -0
- metadata +74 -20
- data/app/cache_processors/qa_server/performance_daily_graph_cache.rb +0 -60
- data/app/cache_processors/qa_server/performance_hourly_graph_cache.rb +0 -65
- data/app/cache_processors/qa_server/performance_monthly_graph_cache.rb +0 -60
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
# Generate graphs for the past 30 days using cached data. Graphs are generated only if the cache has expired.
|
|
3
|
-
module QaServer
|
|
4
|
-
class PerformanceDailyGraphCache
|
|
5
|
-
class_attribute :authority_list_class, :graph_data_service, :graphing_service
|
|
6
|
-
self.authority_list_class = QaServer::AuthorityListerService
|
|
7
|
-
self.graph_data_service = QaServer::PerformanceGraphDataService
|
|
8
|
-
self.graphing_service = QaServer::PerformanceGraphingService
|
|
9
|
-
|
|
10
|
-
class << self
|
|
11
|
-
include QaServer::CacheKeys
|
|
12
|
-
include QaServer::PerformanceHistoryDataKeys
|
|
13
|
-
|
|
14
|
-
# Generates graphs for the past 30 days for :search, :fetch, and :all actions for each authority.
|
|
15
|
-
# @param force [Boolean] if true, run the tests even if the cache hasn't expired; otherwise, use cache if not expired
|
|
16
|
-
def generate_graphs(force: false)
|
|
17
|
-
return unless QaServer::CacheExpiryService.cache_expired?(key: cache_key_for_force, force: force, next_expiry: next_expiry)
|
|
18
|
-
QaServer.config.monitor_logger.debug("(QaServer::PerformanceDailyGraphCache) - GENERATING daily performance graphs (force: #{force})")
|
|
19
|
-
generate_graphs_for_authorities
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
private
|
|
23
|
-
|
|
24
|
-
def generate_graphs_for_authorities
|
|
25
|
-
auths = authority_list_class.authorities_list
|
|
26
|
-
generate_graphs_for_authority(authority_name: ALL_AUTH) # generates graph for all authorities
|
|
27
|
-
auths.each { |authname| generate_graphs_for_authority(authority_name: authname) }
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
def generate_graphs_for_authority(authority_name:)
|
|
31
|
-
[SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
|
|
32
|
-
hash[action] = generate_30_day_graph(authority_name: authority_name, action: action)
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
def generate_30_day_graph(authority_name:, action:)
|
|
37
|
-
# real expiration or force caught by cache_expired? So if we are here, either the cache has expired
|
|
38
|
-
# or force was requested. We still expire the cache and use ttl to catch race conditions.
|
|
39
|
-
Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
|
|
40
|
-
expires_in: next_expiry, race_condition_ttl: 1.hour, force: true) do
|
|
41
|
-
data = graph_data_service.calculate_last_30_days(authority_name: authority_name, action: action)
|
|
42
|
-
graphing_service.generate_daily_graph(authority_name: authority_name, action: action, data: data)
|
|
43
|
-
data
|
|
44
|
-
end
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
def cache_key_for_authority_action(authority_name:, action:)
|
|
48
|
-
"#{PERFORMANCE_GRAPH_DAILY_DATA_CACHE_KEY}--#{authority_name}--#{action}"
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
def cache_key_for_force
|
|
52
|
-
"#{PERFORMANCE_GRAPH_DAILY_DATA_CACHE_KEY}--force"
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
def next_expiry
|
|
56
|
-
QaServer::CacheExpiryService.cache_expiry
|
|
57
|
-
end
|
|
58
|
-
end
|
|
59
|
-
end
|
|
60
|
-
end
|
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
# Generate graphs for the past 24 hours using cached data. The last hour of data is always calculated and all graphs
|
|
3
|
-
# for are generated.
|
|
4
|
-
module QaServer
|
|
5
|
-
class PerformanceHourlyGraphCache
|
|
6
|
-
class_attribute :authority_list_class, :graph_data_service, :graphing_service
|
|
7
|
-
self.authority_list_class = QaServer::AuthorityListerService
|
|
8
|
-
self.graph_data_service = QaServer::PerformanceGraphDataService
|
|
9
|
-
self.graphing_service = QaServer::PerformanceGraphingService
|
|
10
|
-
|
|
11
|
-
class << self
|
|
12
|
-
include QaServer::CacheKeys
|
|
13
|
-
include QaServer::PerformanceHistoryDataKeys
|
|
14
|
-
|
|
15
|
-
# Generates graphs for the past 24 hours for :search, :fetch, and :all actions for each authority.
|
|
16
|
-
# @param force [Boolean] if true, run the tests even if the cache hasn't expired; otherwise, use cache if not expired
|
|
17
|
-
def generate_graphs(force: false)
|
|
18
|
-
QaServer.config.monitor_logger.debug("(QaServer::PerformanceHourlyGraphCache) - GENERATING hourly performance graphs (force: #{force})")
|
|
19
|
-
QaServer.config.performance_cache.write_all
|
|
20
|
-
generate_graphs_for_authorities(force: force)
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
private
|
|
24
|
-
|
|
25
|
-
def generate_graphs_for_authorities(force:)
|
|
26
|
-
auths = authority_list_class.authorities_list
|
|
27
|
-
generate_graphs_for_authority(authority_name: ALL_AUTH, force: force) # generates graph for all authorities
|
|
28
|
-
auths.each { |authname| generate_graphs_for_authority(authority_name: authname, force: force) }
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
def generate_graphs_for_authority(authority_name:, force:)
|
|
32
|
-
[SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
|
|
33
|
-
hash[action] = generate_24_hour_graph(authority_name: authority_name, action: action, force: force)
|
|
34
|
-
end
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
def generate_24_hour_graph(authority_name:, action:, force:)
|
|
38
|
-
graph_created = false
|
|
39
|
-
data = Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
|
|
40
|
-
expires_in: QaServer::TimeService.current_time.end_of_hour - QaServer::TimeService.current_time,
|
|
41
|
-
race_condition_ttl: 1.hour, force: force) do
|
|
42
|
-
data = graph_data_service.calculate_last_24_hours(authority_name: authority_name, action: action)
|
|
43
|
-
graphing_service.generate_hourly_graph(authority_name: authority_name, action: action, data: data)
|
|
44
|
-
graph_created = true
|
|
45
|
-
data
|
|
46
|
-
end
|
|
47
|
-
regen_last_hour_and_graph(authority_name: authority_name, action: action, data: data) unless graph_created
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
def regen_last_hour_and_graph(authority_name:, action:, data:)
|
|
51
|
-
Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
|
|
52
|
-
expires_in: QaServer::TimeService.current_time.end_of_hour - QaServer::TimeService.current_time,
|
|
53
|
-
race_condition_ttl: 1.hour, force: true) do
|
|
54
|
-
data = graph_data_service.recalculate_last_hour(authority_name: authority_name, action: action, averages: data)
|
|
55
|
-
graphing_service.generate_hourly_graph(authority_name: authority_name, action: action, data: data)
|
|
56
|
-
data
|
|
57
|
-
end
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
def cache_key_for_authority_action(authority_name:, action:)
|
|
61
|
-
"#{PERFORMANCE_GRAPH_HOURLY_DATA_CACHE_KEY}--#{authority_name}--#{action}"
|
|
62
|
-
end
|
|
63
|
-
end
|
|
64
|
-
end
|
|
65
|
-
end
|
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
# Generate graphs for the past 12 months using cached data. Graphs are generated only if the cache has expired.
|
|
3
|
-
module QaServer
|
|
4
|
-
class PerformanceMonthlyGraphCache
|
|
5
|
-
class_attribute :authority_list_class, :graph_data_service, :graphing_service
|
|
6
|
-
self.authority_list_class = QaServer::AuthorityListerService
|
|
7
|
-
self.graph_data_service = QaServer::PerformanceGraphDataService
|
|
8
|
-
self.graphing_service = QaServer::PerformanceGraphingService
|
|
9
|
-
|
|
10
|
-
class << self
|
|
11
|
-
include QaServer::CacheKeys
|
|
12
|
-
include QaServer::PerformanceHistoryDataKeys
|
|
13
|
-
|
|
14
|
-
# Generates graphs for the past 30 days for :search, :fetch, and :all actions for each authority.
|
|
15
|
-
# @param force [Boolean] if true, run the tests even if the cache hasn't expired; otherwise, use cache if not expired
|
|
16
|
-
def generate_graphs(force: false)
|
|
17
|
-
return unless QaServer::CacheExpiryService.cache_expired?(key: cache_key_for_force, force: force, next_expiry: next_expiry)
|
|
18
|
-
QaServer.config.monitor_logger.debug("(QaServer::PerformanceMonthlyGraphCache) - GENERATING monthly performance graphs (force: #{force})")
|
|
19
|
-
generate_graphs_for_authorities
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
private
|
|
23
|
-
|
|
24
|
-
def generate_graphs_for_authorities
|
|
25
|
-
auths = authority_list_class.authorities_list
|
|
26
|
-
generate_graphs_for_authority(authority_name: ALL_AUTH) # generates graph for all authorities
|
|
27
|
-
auths.each { |authname| generate_graphs_for_authority(authority_name: authname) }
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
def generate_graphs_for_authority(authority_name:)
|
|
31
|
-
[SEARCH, FETCH, ALL_ACTIONS].each_with_object({}) do |action, hash|
|
|
32
|
-
hash[action] = generate_12_month_graph(authority_name: authority_name, action: action)
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
def generate_12_month_graph(authority_name:, action:)
|
|
37
|
-
# real expiration or force caught by cache_expired? So if we are here, either the cache has expired
|
|
38
|
-
# or force was requested. We still expire the cache and use ttl to catch race conditions.
|
|
39
|
-
Rails.cache.fetch(cache_key_for_authority_action(authority_name: authority_name, action: action),
|
|
40
|
-
expires_in: next_expiry, race_condition_ttl: 1.hour, force: true) do
|
|
41
|
-
data = graph_data_service.calculate_last_12_months(authority_name: authority_name, action: action)
|
|
42
|
-
graphing_service.generate_monthly_graph(authority_name: authority_name, action: action, data: data)
|
|
43
|
-
data
|
|
44
|
-
end
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
def cache_key_for_authority_action(authority_name:, action:)
|
|
48
|
-
"#{PERFORMANCE_GRAPH_MONTHLY_DATA_CACHE_KEY}--#{authority_name}--#{action}"
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
def cache_key_for_force
|
|
52
|
-
"#{PERFORMANCE_GRAPH_MONTHLY_DATA_CACHE_KEY}--force"
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
def next_expiry
|
|
56
|
-
QaServer::CacheExpiryService.cache_expiry
|
|
57
|
-
end
|
|
58
|
-
end
|
|
59
|
-
end
|
|
60
|
-
end
|