qa_server 6.2.0 → 7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +11 -0
  3. data/app/cache_processors/concerns/qa_server/cache_keys.rb +15 -0
  4. data/app/cache_processors/qa_server/cache_expiry_service.rb +33 -0
  5. data/app/cache_processors/qa_server/performance_daily_graph_cache.rb +60 -0
  6. data/app/cache_processors/qa_server/performance_datatable_cache.rb +33 -0
  7. data/app/cache_processors/qa_server/performance_hourly_graph_cache.rb +65 -0
  8. data/app/cache_processors/qa_server/performance_monthly_graph_cache.rb +60 -0
  9. data/app/cache_processors/qa_server/scenario_history_cache.rb +35 -0
  10. data/app/cache_processors/qa_server/scenario_run_cache.rb +28 -0
  11. data/app/cache_processors/qa_server/scenario_run_failures_cache.rb +51 -0
  12. data/app/cache_processors/qa_server/scenario_run_summary_cache.rb +40 -0
  13. data/app/controllers/qa_server/monitor_status_controller.rb +39 -48
  14. data/app/jobs/qa_server/monitor_tests_job.rb +1 -1
  15. data/app/models/qa_server/scenario_run_history.rb +27 -84
  16. data/app/models/qa_server/scenario_run_registry.rb +1 -1
  17. data/app/services/qa_server/performance_datatable_service.rb +6 -9
  18. data/app/services/qa_server/performance_graph_data_service.rb +77 -154
  19. data/app/services/qa_server/performance_graphing_service.rb +34 -47
  20. data/lib/qa_server/version.rb +1 -1
  21. data/spec/{services/qa_server/monitor_cache_service_spec.rb → cache_processors/qa_server/cache_expiry_service_spec.rb} +1 -1
  22. metadata +15 -6
  23. data/app/services/qa_server/monitor_cache_service.rb +0 -22
  24. /data/app/{models → cache_processors}/qa_server/performance_cache.rb +0 -0
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
  # Provide access to the scenario_run_history database table which tracks scenario runs over time.
3
3
  module QaServer
4
- class ScenarioRunHistory < ActiveRecord::Base # rubocop:disable Metrics/ClassLength
4
+ class ScenarioRunHistory < ActiveRecord::Base
5
5
  self.table_name = 'scenario_run_history'
6
6
  belongs_to :scenario_run_registry
7
7
  enum scenario_type: [:connection, :accuracy, :performance], _suffix: :type
@@ -35,78 +35,27 @@ module QaServer
35
35
 
36
36
  # Get a summary of passing/failing tests for a run.
37
37
  # @param scenario_run [QaServer::ScenarioRunRegistry] the run on which to gather statistics
38
- # @param force [Boolean] if true, forces cache to regenerate; otherwise, returns value from cache unless expired
39
38
  # @returns [QaServer::ScenarioRunSummary] statistics on the requested run
40
39
  # @example ScenarioRunSummary includes methods for accessing
41
- # * run_id: 14,
42
- # * run_dt_stamp:
43
- # * authority_count: 22,
44
- # * failing_authority_count: 1
45
- # * passing_scenario_count: 156,
46
- # * failing_scenario_count: 3,
47
- # * total_scenario_count: 159,
48
- def run_summary(scenario_run:, force: false)
49
- Rails.cache.fetch("QaServer::ScenarioRunHistory/#{__method__}", expires_in: QaServer::MonitorCacheService.cache_expiry, race_condition_ttl: 1.minute, force: force) do
50
- QaServer.config.monitor_logger.debug("(QaServer::ScenarioRunHistory##{__method__}) - CALCULATING summary of latest run - cache expired or refresh requested (force: #{force})")
51
- status = status_counts_in_run(run_id: scenario_run.id)
52
- summary_class.new(run_id: scenario_run.id,
53
- run_dt_stamp: scenario_run.dt_stamp,
54
- authority_count: authorities_in_run(run_id: scenario_run.id).count,
55
- failing_authority_count: authorities_with_failures_in_run(run_id: scenario_run.id).count,
56
- passing_scenario_count: status['good'],
57
- failing_scenario_count: status['bad'] + status['unknown'])
58
- end
59
- end
60
-
61
- # Get set of all scenario results for a run.
62
- # @param run_id [Integer] the run on which to gather statistics
63
- # @param authority_name [String] limit results to those for the authority with this name
64
- # @param status [Array<Symbol> | Symbol] :good, :bad, :unknown, or any of these in an array to select multiple status
65
- # @param url [String] limit results to a specific scenario URL
66
- # @returns [Array<ScenarioRunHistory>] scenario details for all scenarios in the run
67
- # @example
68
- # [ { status: :bad,
69
- # authority_name: "geonames_ld4l_cache",
70
- # subauthority_name: "area",
71
- # service: "ld4l_cache",
72
- # action: "search",
73
- # url: "/qa/search/linked_data/geonames_ld4l_cache/area?q=France&maxRecords=4",
74
- # err_message: "Unable to connect to authority",
75
- # scenario_type: :connection
76
- # run_time: 11.2 },
77
- # { status: :good,
78
- # authority_name: "oclcfast_ld4l_cache",
79
- # subauthority_name: "Organization",
80
- # service: "ld4l_cache",
81
- # action: "search",
82
- # url: "/qa/search/linked_data/oclcfast_ld4l_cache/organization?q=mark twain&maxRecords=4",
83
- # err_message: "",
84
- # scenario_type: :connection
85
- # run_time: 0.131 },
86
- # { status: :unknown,
87
- # authority_name: "oclcfast_ld4l_cache",
88
- # subauthority_name: "Person",
89
- # service: "ld4l_cache",
90
- # action: "search",
91
- # url: "/qa/search/linked_data/oclcfast_ld4l_cache/person?q=mark twain&maxRecords=4",
92
- # err_message: "Not enough search results returned",
93
- # scenario_type: :connection
94
- # run_time: 0.123 } ]
95
- # @deprecated Not used anywhere. Being removed.
96
- def run_results(run_id:, authority_name: nil, status: nil, url: nil)
97
- return [] unless run_id
98
- where = {}
99
- where[:scenario_run_registry_id] = run_id
100
- where[:authority_name] = authority_name if authority_name.present?
101
- where[:status] = status if status.present?
102
- where[:url] = url if url.present?
103
- QaServer::ScenarioRunHistory.where(where).to_a
40
+ # * run_id [Integer] e.g. 14
41
+ # * run_dt_stamp [ActiveSupport::TimeWithZone] e.g. Wed, 19 Feb 2020 16:01:07 UTC +00:00
42
+ # * authority_count [Integer] e.g. 22
43
+ # * failing_authority_count [Integer] e.g. 1
44
+ # * passing_scenario_count [Integer] e.g. 156
45
+ # * failing_scenario_count [Integer] e.g. 3
46
+ # * total_scenario_count [Integer] e.g. 159
47
+ def run_summary(scenario_run:)
48
+ status = status_counts_in_run(run_id: scenario_run.id)
49
+ summary_class.new(run_id: scenario_run.id,
50
+ run_dt_stamp: scenario_run.dt_stamp,
51
+ authority_count: authorities_in_run(run_id: scenario_run.id).count,
52
+ failing_authority_count: authorities_with_failures_in_run(run_id: scenario_run.id).count,
53
+ passing_scenario_count: status['good'],
54
+ failing_scenario_count: status['bad'] + status['unknown'])
104
55
  end
105
- deprecation_deprecate run_results: "Not used anywhere. Being removed."
106
56
 
107
57
  # Get set of failures for a run, if any.
108
58
  # @param run_id [Integer] the run on which to gather statistics
109
- # @param force [Boolean] if true, forces cache to regenerate; otherwise, returns value from cache unless expired
110
59
  # @returns [Array<Hash>] scenario details for any failing scenarios in the run
111
60
  # @example
112
61
  # [ { status: :bad,
@@ -127,29 +76,23 @@ module QaServer
127
76
  # err_message: "Not enough search results returned",
128
77
  # scenario_type: :connection
129
78
  # run_time: 0.123 } ]
130
- def run_failures(run_id:, force: false)
79
+ def run_failures(run_id:)
131
80
  return [] unless run_id
132
- Rails.cache.fetch("QaServer::ScenarioRunHistory/#{__method__}", expires_in: QaServer::MonitorCacheService.cache_expiry, race_condition_ttl: 1.minute, force: force) do
133
- QaServer.config.monitor_logger.debug("(QaServer::ScenarioRunHistory##{__method__}) - finding failures in latest run - cache expired or refresh requested (force: #{force})")
134
- QaServer::ScenarioRunHistory.where(scenario_run_registry_id: run_id).where.not(status: :good).to_a
135
- end
81
+ QaServer::ScenarioRunHistory.where(scenario_run_registry_id: run_id).where.not(status: :good).to_a
136
82
  end
137
83
 
138
- # Get a summary level of historical data
139
- # @returns [Array<Array>] summary of passing/failing tests for each authority
84
+ # Get a summary of the number of days passing/failing for scenario runs during configured time period
85
+ # @returns [Array<Hash>] count of days with passing/failing tests for each authority
140
86
  # @example [auth_name, failing, passing]
141
87
  # { 'agrovoc' => { good: 31, bad: 2 },
142
88
  # 'geonames_ld4l_cache' => { good: 32, bad: 1 } }
143
- def historical_summary(force: false)
144
- Rails.cache.fetch("QaServer::ScenarioRunHistory/#{__method__}", expires_in: QaServer::MonitorCacheService.cache_expiry, race_condition_ttl: 1.minute, force: force) do
145
- QaServer.config.monitor_logger.debug("(QaServer::ScenarioRunHistory##{__method__}) - CALCULATING authority connection history - cache expired or refresh requested (force: #{force})")
146
- days_good = count_days(:good)
147
- days_bad = count_days(:bad)
148
- days_unknown = count_days(:unknown)
149
- keys = (days_good.keys + days_bad.keys + days_unknown.keys).uniq.sort
150
- keys.each_with_object({}) do |auth, hash|
151
- hash[auth] = { good: day_count(auth, days_good), bad: day_count(auth, days_bad) + day_count(auth, days_unknown) }
152
- end
89
+ def historical_summary
90
+ days_good = count_days(:good)
91
+ days_bad = count_days(:bad)
92
+ days_unknown = count_days(:unknown)
93
+ keys = (days_good.keys + days_bad.keys + days_unknown.keys).uniq.sort
94
+ keys.each_with_object({}) do |auth, hash|
95
+ hash[auth] = { good: day_count(auth, days_good), bad: day_count(auth, days_bad) + day_count(auth, days_unknown) }
153
96
  end
154
97
  end
155
98
 
@@ -25,7 +25,7 @@ module QaServer
25
25
 
26
26
  # @return [ActiveSupport::TimeWithZone] datetime stamp of first registered run
27
27
  def self.first_run_dt
28
- Rails.cache.fetch("#{self.class}/#{__method__}", expires_in: QaServer::MonitorCacheService.cache_expiry, race_condition_ttl: 30.seconds) do
28
+ Rails.cache.fetch("#{self.class}/#{__method__}", expires_in: QaServer::CacheExpiryService.cache_expiry, race_condition_ttl: 30.seconds) do
29
29
  QaServer::ScenarioRunRegistry.first.dt_stamp
30
30
  end
31
31
  end
@@ -11,7 +11,6 @@ module QaServer
11
11
  self.authority_list_class = QaServer::AuthorityListerService
12
12
 
13
13
  # Summary of performance by action for each authority for the configured time period (e.g. :day, :month, :year, :all).
14
- # @param force [Boolean] if true, calculate the stats even if the cache hasn't expired; otherwise, use cache if not expired
15
14
  # @returns [Hash] performance statistics for configured time period by action for each authority
16
15
  # @example
17
16
  # { all_authorities:
@@ -24,14 +23,12 @@ module QaServer
24
23
  # },
25
24
  # AGROVOC_LD4L_CACHE: { ... # same data for each authority }
26
25
  # }
27
- def calculate_datatable_data(force:)
28
- Rails.cache.fetch("QaServer::PerformanceDatatableService/#{__method__}", expires_in: QaServer::MonitorCacheService.cache_expiry, race_condition_ttl: 5.minutes, force: force) do
29
- QaServer.config.monitor_logger.debug("(QaServer::PerformanceDatatableService##{__method__}) - CALCULATING performance datatable stats - cache expired or refresh requested (force: #{force})")
30
- data = {}
31
- auths = authority_list_class.authorities_list
32
- data[ALL_AUTH] = datatable_data_for_authority
33
- auths.each { |auth_name| data[auth_name] = datatable_data_for_authority(authority_name: auth_name) }
34
- data
26
+ def calculate_datatable_data
27
+ data = {}
28
+ auths = authority_list_class.authorities_list
29
+ data[ALL_AUTH] = datatable_data_for_authority
30
+ auths.each_with_object(data) do |auth_name, data| # rubocop:disable Lint/ShadowingOuterLocalVariable
31
+ data[auth_name] = datatable_data_for_authority(authority_name: auth_name)
35
32
  end
36
33
  end
37
34
 
@@ -1,135 +1,99 @@
1
1
  # frozen_string_literal: true
2
- # This class sets performance stats for the last 24 hours, past 30 days, and the past 12 months.
2
+ # This class calculates performance averages to be used to generate graphs for the last 24 hours, 30 days, and 12 months.
3
3
  module QaServer
4
- class PerformanceGraphDataService # rubocop:disable Metrics/ClassLength
4
+ class PerformanceGraphDataService
5
5
  class << self
6
6
  include QaServer::PerformanceHistoryDataKeys
7
7
 
8
- class_attribute :stats_calculator_class, :performance_data_class, :authority_list_class
8
+ class_attribute :stats_calculator_class, :performance_data_class
9
9
  self.stats_calculator_class = QaServer::PerformanceCalculatorService
10
10
  self.performance_data_class = QaServer::PerformanceHistory
11
- self.authority_list_class = QaServer::AuthorityListerService
12
11
 
13
- # Performance data for a day, a month, a year, and all time for each authority.
14
- # @param datatype [Symbol] what type of data should be calculated (e.g. :datatable, :graph, :all)
15
- # @returns [Hash] performance statistics for the past 24 hours
16
- # @example
17
- # { all_authorities:
18
- # { search:
19
- # {
20
- # day:
21
- # { 0: { hour: '1400', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
22
- # 1: { hour: '1500', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
23
- # 2: { hour: '1600', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
24
- # ...,
25
- # 23: { hour: 'NOW', retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
26
- # },
27
- # month:
28
- # { 0: { day: '07-15-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
29
- # 1: { day: '07-16-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
30
- # 2: { day: '07-17-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
31
- # ...,
32
- # 29: { day: 'TODAY', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
33
- # },
34
- # year:
35
- # { 0: { month: '09-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
36
- # 1: { month: '10-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
37
- # 2: { month: '11-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
38
- # ...,
39
- # 11: { month: '08-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
40
- # }
41
- # },
42
- # fetch: { ... # same data as for search_stats },
43
- # all: { ... # same data as for search_stats }
44
- # },
45
- # AGROVOC_LD4L_CACHE: { ... # same data for each authority }
12
+ # Performance data for the last 12 months for a specific authority and action
13
+ # @param authority_name [String] name of an authority
14
+ # @param action [Symbol] :search, :fetch, or :all_actions
15
+ # @returns [Hash] performance statistics for the past 12 months
16
+ # @example returns
17
+ # { 0: { month: '09-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
18
+ # 1: { month: '10-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
19
+ # 2: { month: '11-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
20
+ # ...,
21
+ # 11: { month: '08-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
46
22
  # }
47
- def calculate_graph_data(force:)
48
- QaServer.config.performance_cache.write_all
49
- data = {}
50
- auths = authority_list_class.authorities_list
51
- calculate_all = force || cache_expired?
52
- QaServer.config.monitor_logger.debug("(QaServer::PerformanceGraphDataService##{__method__}) - CALCULATING performance graph data (calculate_all: #{calculate_all})")
53
- data[ALL_AUTH] = graph_data_for_authority(force: force, calculate_all: calculate_all)
54
- auths.each { |auth_name| data[auth_name] = graph_data_for_authority(authority_name: auth_name, force: force, calculate_all: calculate_all) }
55
- data
23
+ def calculate_last_12_months(authority_name:, action:)
24
+ start_month = QaServer::TimeService.current_time.beginning_of_month - 11.months
25
+ 0.upto(11).each_with_object({}) do |idx, averages|
26
+ records = records_by(authority_name, action, start_month..start_month.end_of_month)
27
+ averages[idx] = calculate_from_records(records, BY_MONTH, start_month.strftime("%m-%Y"))
28
+ start_month += 1.month
29
+ end
56
30
  end
57
31
 
58
- private
59
-
60
- def graph_data_for_authority(authority_name: nil, force:, calculate_all:)
61
- [:search, :fetch, :all_actions].each_with_object({}) do |action, hash|
62
- data = {}
63
- data[FOR_DAY] = average_last_24_hours(authority_name: authority_name, action: action, force: force)
64
- data[FOR_MONTH] = average_last_30_days(authority_name: authority_name, action: action, force: force) if calculate_all
65
- data[FOR_YEAR] = average_last_12_months(authority_name: authority_name, action: action, force: force) if calculate_all
66
- hash[action] = data
67
- end
32
+ # Performance data for the last 30 days for a specific authority and action
33
+ # @param authority_name [String] name of an authority
34
+ # @param action [Symbol] :search, :fetch, or :all_actions
35
+ # @returns [Hash] performance statistics for the past 30 days
36
+ # @example returns
37
+ # { 0: { day: '07-15-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
38
+ # 1: { day: '07-16-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
39
+ # 2: { day: '07-17-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
40
+ # ...,
41
+ # 29: { day: 'TODAY', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
42
+ # }
43
+ def calculate_last_30_days(authority_name:, action:)
44
+ start_day = QaServer::TimeService.current_time.beginning_of_day - 29.days
45
+ 0.upto(29).each_with_object({}) do |idx, averages|
46
+ records = records_by(authority_name, action, start_day..start_day.end_of_day)
47
+ averages[idx] = calculate_from_records(records, BY_DAY, performance_by_day_label(idx, start_day))
48
+ start_day += 1.day
68
49
  end
50
+ end
69
51
 
70
- # Get hourly average for the past 24 hours.
71
- # @param authority_name [String] limit statistics to records for the given authority (default: all authorities)
72
- # @param action [Symbol] one of :search, :fetch, :all_actions
73
- # @param force [Boolean] if true, forces cache to regenerate; otherwise, returns value from cache unless expired
74
- # @returns [Hash] performance statistics for the past 24 hours
75
- # @example
76
- # { 0: { hour: '1400', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
77
- # 1: { hour: '1500', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
78
- # 2: { hour: '1600', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
79
- # ...,
80
- # 23: { hour: 'NOW', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
81
- # }
82
- def average_last_24_hours(authority_name: nil, action: nil, force: false)
83
- avgs = Rails.cache.fetch("QaServer::PerformanceGraphDataService/#{__method__}/#{authority_name || ALL_AUTH}/#{action}/#{FOR_DAY}",
84
- expires_in: QaServer::TimeService.current_time.end_of_hour - QaServer::TimeService.current_time,
85
- race_condition_ttl: 1.hour, force: force) do
86
- calculate_last_24_hours(authority_name, action)
87
- end
88
- calculate_last_hour(authority_name, action, avgs)
52
+ # Performance data for the last 24 hours for a specific authority and action
53
+ # @param authority_name [String] name of an authority
54
+ # @param action [Symbol] :search, :fetch, or :all_actions
55
+ # @returns [Hash] performance statistics for the past 24 hours
56
+ # @example returns
57
+ # { 0: { hour: '1400', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
58
+ # 1: { hour: '1500', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
59
+ # 2: { hour: '1600', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
60
+ # ...,
61
+ # 23: { hour: 'NOW', retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
62
+ # }
63
+ def calculate_last_24_hours(authority_name:, action:)
64
+ start_hour = QaServer::TimeService.current_time.beginning_of_hour - 23.hours
65
+ 0.upto(23).each_with_object({}) do |idx, averages|
66
+ records = records_by(authority_name, action, start_hour..start_hour.end_of_hour)
67
+ averages[idx] = calculate_from_records(records, BY_HOUR, performance_by_hour_label(idx, start_hour))
68
+ start_hour += 1.hour
89
69
  end
70
+ end
90
71
 
91
- # Get daily average for the past 30 days.
92
- # @param authority_name [String] limit statistics to records for the given authority (default: all authorities)
93
- # @param action [Symbol] one of :search, :fetch, :all_actions
94
- # @param force [Boolean] if true, forces cache to regenerate; otherwise, returns value from cache unless expired
95
- # @returns [Hash] performance statistics for the past 30 days
96
- # @example
97
- # { 0: { day: '07-15-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
98
- # 1: { day: '07-16-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
99
- # 2: { day: '07-17-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
100
- # ...,
101
- # 29: { day: 'TODAY', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
102
- # }
103
- def average_last_30_days(authority_name: nil, action: nil, force: false)
104
- Rails.cache.fetch("QaServer::PerformanceGraphDataService/#{__method__}/#{authority_name || ALL_AUTH}/#{action}/#{FOR_MONTH}",
105
- expires_in: QaServer::MonitorCacheService.cache_expiry, race_condition_ttl: 1.hour, force: force) do
106
- calculate_last_30_days(authority_name, action)
107
- end
108
- end
72
+ # Performance data for the last 24 hours for a specific authority and action
73
+ # @param authority_name [String] name of an authority
74
+ # @param action [Symbol] :search, :fetch, or :all_actions
75
+ # @param averages [Hash] existing data for each hour
76
+ # @returns [Hash] existing hourly data with the last hour updated
77
+ # @example returns
78
+ # { 0: { hour: '1400', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
79
+ # 1: { hour: '1500', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
80
+ # 2: { hour: '1600', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
81
+ # ...,
82
+ # 23: { hour: 'NOW', retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
83
+ # }
84
+ def recalculate_last_hour(authority_name:, action:, averages:)
85
+ start_hour = QaServer::TimeService.current_time.beginning_of_hour
86
+ records = records_by(authority_name, action, start_hour..start_hour.end_of_hour)
87
+ averages[23] = calculate_from_records(records, BY_HOUR, performance_by_hour_label(23, start_hour))
88
+ averages
89
+ end
109
90
 
110
- # Get daily average for the past 12 months.
111
- # @param authority_name [String] limit statistics to records for the given authority (default: all authorities)
112
- # @param action [Symbol] one of :search, :fetch, :all_actions
113
- # @param force [Boolean] if true, forces cache to regenerate; otherwise, returns value from cache unless expired
114
- # @returns [Hash] performance statistics for the past 12 months
115
- # @example
116
- # { 0: { month: '09-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
117
- # 1: { month: '10-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
118
- # 2: { month: '11-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
119
- # ...,
120
- # 11: { month: '08-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
121
- # }
122
- def average_last_12_months(authority_name: nil, action: nil, force: false)
123
- Rails.cache.fetch("QaServer::PerformanceGraphDataService/#{__method__}/#{authority_name || ALL_AUTH}/#{action}/#{FOR_YEAR}",
124
- expires_in: QaServer::MonitorCacheService.cache_expiry, race_condition_ttl: 1.hour, force: force) do
125
- calculate_last_12_months(authority_name, action)
126
- end
127
- end
91
+ private
128
92
 
129
93
  def records_by(authority_name, action, time_period)
130
94
  where_clause = { dt_stamp: time_period }
131
- where_clause[:authority] = authority_name unless authority_name.nil?
132
- where_clause[:action] = action unless action.nil? || action == :all_actions
95
+ where_clause[:authority] = authority_name unless authority_name.nil? || authority_name == ALL_AUTH
96
+ where_clause[:action] = action unless action.nil? || action == ALL_ACTIONS
133
97
  performance_data_class.where(where_clause)
134
98
  end
135
99
 
@@ -157,47 +121,6 @@ module QaServer
157
121
  stats = stats_calculator_class.new(records).calculate_average_stats
158
122
  { STATS => stats, range_idx => range_label }
159
123
  end
160
-
161
- def calculate_last_hour(authority_name, action, avgs)
162
- start_hour = QaServer::TimeService.current_time.beginning_of_hour
163
- records = records_by(authority_name, action, start_hour..start_hour.end_of_hour)
164
- avgs[23] = calculate_from_records(records, BY_HOUR, performance_by_hour_label(23, start_hour))
165
- avgs
166
- end
167
-
168
- def calculate_last_24_hours(authority_name, action)
169
- start_hour = QaServer::TimeService.current_time.beginning_of_hour - 23.hours
170
- 0.upto(23).each_with_object({}) do |idx, avgs|
171
- records = records_by(authority_name, action, start_hour..start_hour.end_of_hour)
172
- avgs[idx] = calculate_from_records(records, BY_HOUR, performance_by_hour_label(idx, start_hour))
173
- start_hour += 1.hour
174
- end
175
- end
176
-
177
- def calculate_last_30_days(authority_name, action)
178
- start_day = QaServer::TimeService.current_time.beginning_of_day - 29.days
179
- 0.upto(29).each_with_object({}) do |idx, avgs|
180
- records = records_by(authority_name, action, start_day..start_day.end_of_day)
181
- avgs[idx] = calculate_from_records(records, BY_DAY, performance_by_day_label(idx, start_day))
182
- start_day += 1.day
183
- end
184
- end
185
-
186
- def calculate_last_12_months(authority_name, action)
187
- start_month = QaServer::TimeService.current_time.beginning_of_month - 11.months
188
- 0.upto(11).each_with_object({}) do |idx, avgs|
189
- records = records_by(authority_name, action, start_month..start_month.end_of_month)
190
- avgs[idx] = calculate_from_records(records, BY_MONTH, start_month.strftime("%m-%Y"))
191
- start_month += 1.month
192
- end
193
- end
194
-
195
- # @returns [Boolean] true if cache has expired; otherwise, false
196
- def cache_expired?
197
- expired = Rails.cache.fetch("QaServer::PerformanceGraphDataService/#{__method__}", expires_in: 5.seconds) { true }
198
- Rails.cache.fetch("QaServer::PerformanceGraphDataService/#{__method__}", expires_in: QaServer::MonitorCacheService.cache_expiry, force: expired) { false } # reset if expired
199
- expired
200
- end
201
124
  end
202
125
  end
203
126
  end
@@ -9,17 +9,6 @@ module QaServer
9
9
  class_attribute :authority_list_class
10
10
  self.authority_list_class = QaServer::AuthorityListerService
11
11
 
12
- # @param performance_data [Hash] hash of all performance data for all authorities
13
- # @see QaServer:PerformanceHistory
14
- def create_performance_graphs(performance_data:)
15
- QaServer.config.monitor_logger.debug("(QaServer::PerformanceGraphingService##{__method__}) - generating graphs")
16
- performance_data.each_key do |auth_name|
17
- create_graphs_for_authority(performance_data, auth_name.to_sym, :search)
18
- create_graphs_for_authority(performance_data, auth_name.to_sym, :fetch)
19
- create_graphs_for_authority(performance_data, auth_name.to_sym, :all_actions)
20
- end
21
- end
22
-
23
12
  # @param authority_name [String] name of the authority
24
13
  # @param action [Symbol] action performed by the request (e.g. :search, :fetch, :all_actions)
25
14
  # @param time_period [Symbol] time period for the graph (i.e. :day, :month, :year)
@@ -27,45 +16,43 @@ module QaServer
27
16
  File.join(graph_relative_path, graph_filename(authority_name, action, time_period))
28
17
  end
29
18
 
30
- private
31
-
32
- def create_graphs_for_authority(performance_data, authority_name, action)
33
- create_performance_for_day_graph(performance_data, authority_name, action)
34
- create_performance_for_month_graph(performance_data, authority_name, action)
35
- create_performance_for_year_graph(performance_data, authority_name, action)
36
- end
37
-
38
- def create_performance_for_day_graph(performance_data, authority_name, action)
39
- data = authority_performance_data(performance_data, authority_name, action, FOR_DAY)
40
- return if data.empty?
41
- gruff_data = rework_performance_data_for_gruff(data, BY_HOUR)
42
- create_gruff_graph(gruff_data,
43
- performance_for_day_graph_full_path(authority_name, action),
44
- I18n.t('qa_server.monitor_status.performance.x_axis_hour'))
45
- end
19
+ # Generate one 12 month graph for the authority and action given the graph data.
20
+ # @param authority_name [String] name of the authority
21
+ # @param action [Symbol] action performed by the request (e.g. :search, :fetch, :all_actions)
22
+ # @param data [Hash] data to use to generate the graph
23
+ # @see QaServer::PerformanceGraphDataService.calculate_last_12_months
24
+ def generate_monthly_graph(authority_name: ALL_AUTH, action:, data:)
25
+ gruff_data = rework_performance_data_for_gruff(data, BY_MONTH)
26
+ create_gruff_graph(gruff_data,
27
+ performance_for_year_graph_full_path(authority_name, action),
28
+ I18n.t('qa_server.monitor_status.performance.x_axis_month'))
29
+ end
46
30
 
47
- def create_performance_for_month_graph(performance_data, authority_name, action)
48
- data = authority_performance_data(performance_data, authority_name, action, FOR_MONTH)
49
- return if data.empty?
50
- gruff_data = rework_performance_data_for_gruff(data, BY_DAY)
51
- create_gruff_graph(gruff_data,
52
- performance_for_month_graph_full_path(authority_name, action),
53
- I18n.t('qa_server.monitor_status.performance.x_axis_day'))
54
- end
31
+ # Generate one 30 day graph for the authority and action given the graph data.
32
+ # @param authority_name [String] name of the authority
33
+ # @param action [Symbol] action performed by the request (e.g. :search, :fetch, :all_actions)
34
+ # @param data [Hash] data to use to generate the graph
35
+ # @see QaServer::PerformanceGraphDataService.calculate_last_30_days
36
+ def generate_daily_graph(authority_name: ALL_AUTH, action:, data:)
37
+ gruff_data = rework_performance_data_for_gruff(data, BY_DAY)
38
+ create_gruff_graph(gruff_data,
39
+ performance_for_month_graph_full_path(authority_name, action),
40
+ I18n.t('qa_server.monitor_status.performance.x_axis_day'))
41
+ end
55
42
 
56
- def create_performance_for_year_graph(performance_data, authority_name, action)
57
- data = authority_performance_data(performance_data, authority_name, action, FOR_YEAR)
58
- return if data.empty?
59
- gruff_data = rework_performance_data_for_gruff(data, BY_MONTH)
60
- create_gruff_graph(gruff_data,
61
- performance_for_year_graph_full_path(authority_name, action),
62
- I18n.t('qa_server.monitor_status.performance.x_axis_month'))
63
- end
43
+ # Generate one 24 hour graph for the authority and action given the graph data.
44
+ # @param authority_name [String] name of the authority
45
+ # @param action [Symbol] action performed by the request (e.g. :search, :fetch, :all_actions)
46
+ # @param data [Hash] data to use to generate the graph
47
+ # @see QaServer::PerformanceGraphDataService.calculate_last_24_hours
48
+ def generate_hourly_graph(authority_name: ALL_AUTH, action:, data:)
49
+ gruff_data = rework_performance_data_for_gruff(data, BY_HOUR)
50
+ create_gruff_graph(gruff_data,
51
+ performance_for_day_graph_full_path(authority_name, action),
52
+ I18n.t('qa_server.monitor_status.performance.x_axis_hour'))
53
+ end
64
54
 
65
- def authority_performance_data(data, authority_name, action, time_period)
66
- auth_name = authority_name.nil? ? ALL_AUTH : authority_name
67
- data[auth_name][action].key?(time_period) ? data[auth_name][action][time_period] : {}
68
- end
55
+ private
69
56
 
70
57
  def performance_for_day_graph_full_path(authority_name, action)
71
58
  graph_full_path(graph_filename(authority_name, action, :day))
@@ -1,4 +1,4 @@
1
1
  # frozen_string_literal: true
2
2
  module QaServer
3
- VERSION = '6.2.0'
3
+ VERSION = '7.0.0'
4
4
  end
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
  require 'spec_helper'
3
3
 
4
- RSpec.describe QaServer::MonitorCacheService do
4
+ RSpec.describe QaServer::CacheExpiryService do
5
5
  # rubocop:disable RSpec/MessageChain
6
6
  let(:timezone_name) { 'Eastern Time (US & Canada)' }
7
7
  before { allow(described_class).to receive_message_chain(:config, :preferred_time_zone_name).and_return(timezone_name) }