qa_server 2.2.3 → 3.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/app/assets/stylesheets/qa_server/_authorities.scss +4 -1
  4. data/app/assets/stylesheets/qa_server/_check-status.scss +5 -0
  5. data/app/assets/stylesheets/qa_server/_monitor-status.scss +2 -2
  6. data/app/models/concerns/qa_server/performance_history_data_keys.rb +16 -3
  7. data/app/models/qa_server/performance_history.rb +54 -46
  8. data/app/prepends/prepended_linked_data/find_term.rb +51 -9
  9. data/app/prepends/prepended_linked_data/search_query.rb +37 -9
  10. data/app/prepends/prepended_rdf/rdf_graph.rb +55 -0
  11. data/app/presenters/concerns/qa_server/monitor_status/performance_datatable_behavior.rb +55 -21
  12. data/app/presenters/concerns/qa_server/monitor_status/performance_graph_behavior.rb +54 -25
  13. data/app/presenters/qa_server/monitor_status_presenter.rb +11 -8
  14. data/app/services/qa_server/performance_calculator_service.rb +50 -47
  15. data/app/services/qa_server/performance_graph_data_service.rb +13 -12
  16. data/app/services/qa_server/performance_graphing_service.rb +49 -40
  17. data/app/views/qa_server/monitor_status/_performance.html.erb +107 -32
  18. data/config/locales/qa_server.en.yml +9 -2
  19. data/lib/generators/qa_server/config_generator.rb +1 -0
  20. data/lib/generators/qa_server/templates/config/authorities/linked_data/agrovoc_ld4l_cache.json +8 -1
  21. data/lib/generators/qa_server/templates/config/authorities/linked_data/dbpedia_ld4l_cache.json +8 -1
  22. data/lib/generators/qa_server/templates/config/authorities/linked_data/geonames_ld4l_cache.json +11 -1
  23. data/lib/generators/qa_server/templates/config/authorities/linked_data/getty_aat_ld4l_cache.json +8 -1
  24. data/lib/generators/qa_server/templates/config/authorities/linked_data/getty_tgn_ld4l_cache.json +8 -1
  25. data/lib/generators/qa_server/templates/config/authorities/linked_data/getty_ulan_ld4l_cache.json +8 -1
  26. data/lib/generators/qa_server/templates/config/authorities/linked_data/locdemographics_ld4l_cache.json +2 -1
  27. data/lib/generators/qa_server/templates/config/authorities/linked_data/locnames_ld4l_cache.json +51 -42
  28. data/lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo_ld4l_cache.json +237 -0
  29. data/lib/generators/qa_server/templates/config/authorities/linked_data/locperformance_ld4l_cache.json +75 -3
  30. data/lib/generators/qa_server/templates/config/authorities/linked_data/mesh_nlm_ld4l_cache.json +8 -1
  31. data/lib/generators/qa_server/templates/config/authorities/linked_data/nalt_ld4l_cache.json +8 -1
  32. data/lib/generators/qa_server/templates/config/authorities/linked_data/oclcfast_ld4l_cache.json +8 -1
  33. data/lib/generators/qa_server/templates/config/authorities/linked_data/rda_registry_ld4l_cache.json +193 -0
  34. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_ld4l_cache_validation.yml +3 -18
  35. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_rwo_ld4l_cache_validation.yml +27 -0
  36. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locperformance_ld4l_cache_validation.yml +1 -0
  37. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/rda_registry_ld4l_cache_validation.yml +255 -0
  38. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/wikidata_direct_validation.yml +6 -0
  39. data/lib/generators/qa_server/templates/config/authorities/linked_data/wikidata_direct.json +32 -0
  40. data/lib/generators/qa_server/templates/config/initializers/qa_server.rb +14 -5
  41. data/lib/generators/qa_server/templates/db/migrate/20191007134527_update_performance_history_table.rb.erb +19 -0
  42. data/lib/qa_server/configuration.rb +27 -5
  43. data/lib/qa_server/version.rb +1 -1
  44. metadata +10 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: a526113eb2d57af9e670446c76d9c92e33cdb1c8
4
- data.tar.gz: b3f9253e899f8ff605632952f8ef5990a17e9147
3
+ metadata.gz: c54a8298febf924d051837fef371b727c32912f7
4
+ data.tar.gz: 5c038a7a516bd50834670fd393ee215086708b73
5
5
  SHA512:
6
- metadata.gz: d19ffc6e5baf8ebf1d742fb6edc06fc9e620eadd8182a8632653e576fcee0d673b65f50eba97e0c345494b0d2970deb2f174f9583f003bd324934045c888c831
7
- data.tar.gz: afe7fa02263b9b375da202bc87dfdb27d7f790653840a398d28630cfbd3f75d40c944485179260a94dd8a6561fa179a367763607954415cccd9feef86f91958f
6
+ metadata.gz: a36a6e1d5072d3978a18a34b84bf338b9d318c7db8948a6fb8bbb3ae2506714e8281b4dfa42faae61f5410a3379d38c2e2c2077d848c14a8895ce9dfb7193fb8
7
+ data.tar.gz: 1a5df12683d02f30c8bccc9f96d721faaffe06484223fe3ae543dc94f2948c11c164fa93a9833c57b0d523b898de54b19a02b344d5a56f9018d27f2a1b3a8304
data/CHANGELOG.md CHANGED
@@ -1,3 +1,11 @@
1
+ ### 3.0.0 (2019-10-09)
2
+
3
+ * refactor performance data and graphs to include stats by action as well as by authority and time period
4
+ * adds pagination start_record parameter for LD4P cached data
5
+ * adds RDA Registry authority
6
+ * splits LCNAF into 2 authorities for LD4P cached data focused locnames and real world object focused locnames_rwo
7
+ * adds wikidata authority - NOTE: The first pass on fetching through this authority is non-performant.
8
+
1
9
  ### 2.2.3 (2019-09-11)
2
10
 
3
11
  * do not process and store performance data if it wasn't returned in the result
@@ -1,3 +1,6 @@
1
- td.table_subheading {
1
+ th.table_subheading, td.table_subheading {
2
2
  background-color: lightgrey;
3
3
  }
4
+ th.table_subheading {
5
+ text-align: center;
6
+ }
@@ -73,6 +73,11 @@ td.position {
73
73
  background-color: #ffffff;
74
74
  }
75
75
 
76
+ .status-not-supported {
77
+ text-align: center;
78
+ background-color: #F3EEEE;
79
+ }
80
+
76
81
  label.horizontal-list {
77
82
  padding-right: 15px;
78
83
  }
@@ -20,11 +20,11 @@ div.performance-data-section-visible {
20
20
  display: block;
21
21
  }
22
22
 
23
- div.right-menu-section {
23
+ div.performance-graph-menu-section {
24
24
  padding-top: 50px;
25
25
  }
26
26
 
27
- ul.right-menu {
27
+ ul.time-period-menu, ul.action-menu {
28
28
  list-style-type: none;
29
29
  li.clickable {
30
30
  display: inline;
@@ -4,6 +4,10 @@
4
4
  module QaServer
5
5
  module PerformanceHistoryDataKeys
6
6
  ALL_AUTH = :all_authorities
7
+ SEARCH = :search
8
+ FETCH = :fetch
9
+ ALL_ACTIONS = :all_actions
10
+
7
11
  STATS = :stats
8
12
  FOR_DATATABLE = :datatable_stats
9
13
 
@@ -17,13 +21,22 @@ module QaServer
17
21
  BY_MONTH = :month
18
22
 
19
23
  LOW_LOAD = :low_load_ms
24
+ LOW_RETR = :low_retrieve_ms
25
+ LOW_GRPH = :low_load_graph_ms
20
26
  LOW_NORM = :low_normalization_ms
27
+ LOW_ACTN = :low_action_request_ms
21
28
  LOW_FULL = :low_full_request_ms
22
29
  AVG_LOAD = :avg_load_ms
30
+ AVG_RETR = :avg_retrieve_ms
31
+ AVG_GRPH = :avg_load_graph_ms
23
32
  AVG_NORM = :avg_normalization_ms
33
+ AVG_ACTN = :avg_action_request_ms
24
34
  AVG_FULL = :avg_full_request_ms
25
- HIGH_LOAD = :max_load_ms
26
- HIGH_NORM = :max_normalization_ms
27
- HIGH_FULL = :max_full_request_ms
35
+ HIGH_LOAD = :high_load_ms
36
+ HIGH_RETR = :high_retrieve_ms
37
+ HIGH_GRPH = :high_load_graph_ms
38
+ HIGH_NORM = :high_normalization_ms
39
+ HIGH_ACTN = :high_action_request_ms
40
+ HIGH_FULL = :high_full_request_ms
28
41
  end
29
42
  end
@@ -16,15 +16,13 @@ module QaServer
16
16
  include QaServer::PerformanceHistoryDataKeys
17
17
 
18
18
  # Save a scenario result
19
- # @param run_id [Integer] the run on which to gather statistics
20
- # @param result [Hash] the scenario result to be saved
21
- def save_result(dt_stamp:, authority:, action:, size_bytes:, load_time_ms:, normalization_time_ms:) # rubocop:disable Metrics/ParameterLists
22
- create(dt_stamp: dt_stamp,
19
+ # @param authority [String] name of the authority
20
+ # @param action [Symbol] type of action being evaluated (e.g. :fetch, :search)
21
+ # @return ActveRecord::Base for the new performance history record
22
+ def create_record(authority:, action:)
23
+ create(dt_stamp: Time.now.getlocal,
23
24
  authority: authority,
24
- action: action,
25
- size_bytes: size_bytes,
26
- load_time_ms: load_time_ms,
27
- normalization_time_ms: normalization_time_ms)
25
+ action: action)
28
26
  end
29
27
 
30
28
  # Performance data for a day, a month, a year, and all time for each authority.
@@ -32,34 +30,38 @@ module QaServer
32
30
  # @returns [Hash] performance statistics for the past 24 hours
33
31
  # @example
34
32
  # { all_authorities:
35
- # { datatable_stats:
36
- # { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }
37
- # }
38
- # { day:
39
- # { 0: { hour: '1400', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
40
- # 1: { hour: '1500', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
41
- # 2: { hour: '1600', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
42
- # ...,
43
- # 23: { hour: 'NOW', load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
44
- # }
45
- # }
46
- # { month:
47
- # { 0: { day: '07-15-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
48
- # 1: { day: '07-16-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
49
- # 2: { day: '07-17-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
50
- # ...,
51
- # 29: { day: 'TODAY', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
52
- # }
53
- # }
54
- # { year:
55
- # { 0: { month: '09-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
56
- # 1: { month: '10-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
57
- # 2: { month: '11-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
58
- # ...,
59
- # 11: { month: '08-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
60
- # }
61
- # }
62
- # { AGROVOC_LD4L_CACHE: ... # same data for each authority }
33
+ # { search:
34
+ # {
35
+ # datatable_stats:
36
+ # { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5,
37
+ # retrieve_10th_ms: 12.3, graph_load_10th_ms: 12.3, normalization_10th_ms: 4.2, full_request_10th_ms: 16.5,
38
+ # retrieve_90th_ms: 12.3, graph_load_90th_ms: 12.3, normalization_90th_ms: 4.2, full_request_90th_ms: 16.5 }
39
+ # day:
40
+ # { 0: { hour: '1400', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
41
+ # 1: { hour: '1500', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
42
+ # 2: { hour: '1600', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
43
+ # ...,
44
+ # 23: { hour: 'NOW', retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
45
+ # },
46
+ # month:
47
+ # { 0: { day: '07-15-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
48
+ # 1: { day: '07-16-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
49
+ # 2: { day: '07-17-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
50
+ # ...,
51
+ # 29: { day: 'TODAY', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
52
+ # },
53
+ # year:
54
+ # { 0: { month: '09-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
55
+ # 1: { month: '10-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
56
+ # 2: { month: '11-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
57
+ # ...,
58
+ # 11: { month: '08-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
59
+ # }
60
+ # },
61
+ # fetch: { ... # same data as for search_stats }
62
+ # all: { ... # same data as for search_stats }
63
+ # },
64
+ # AGROVOC_LD4L_CACHE: { ... # same data for each authority }
63
65
  # }
64
66
  def performance_data(datatype: :datatable)
65
67
  return if datatype == :none
@@ -87,27 +89,33 @@ module QaServer
87
89
  end
88
90
 
89
91
  def data_for_authority(authority_name: nil, datatype:)
90
- data = {}
91
- data[FOR_DATATABLE] = data_table_stats(authority_name) if calculate_datatable?(datatype)
92
- if calculate_graphdata?(datatype)
93
- data[FOR_DAY] = graph_data_service_class.average_last_24_hours(authority_name)
94
- data[FOR_MONTH] = graph_data_service_class.average_last_30_days(authority_name)
95
- data[FOR_YEAR] = graph_data_service_class.average_last_12_months(authority_name)
92
+ action_data = {}
93
+ [:search, :fetch, :all_actions].each do |action|
94
+ data = {}
95
+ data[FOR_DATATABLE] = data_table_stats(authority_name, action) if calculate_datatable?(datatype)
96
+ if calculate_graphdata?(datatype)
97
+ data[FOR_DAY] = graph_data_service_class.average_last_24_hours(authority_name: authority_name, action: action)
98
+ data[FOR_MONTH] = graph_data_service_class.average_last_30_days(authority_name: authority_name, action: action)
99
+ data[FOR_YEAR] = graph_data_service_class.average_last_12_months(authority_name: authority_name, action: action)
100
+ end
101
+ action_data[action] = data
96
102
  end
97
- data
103
+ action_data
98
104
  end
99
105
 
100
106
  # Get statistics for all available data.
101
107
  # @param [String] auth_name - limit statistics to records for the given authority (default: all authorities)
102
108
  # @returns [Hash] performance statistics for the datatable during the expected time period
103
109
  # @example
104
- # { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }
105
- def data_table_stats(auth_name)
110
+ # { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5,
111
+ # retrieve_10th_ms: 12.3, graph_load_10th_ms: 12.3, normalization_10th_ms: 4.2, full_request_10th_ms: 16.5,
112
+ # retrieve_90th_ms: 12.3, graph_load_90th_ms: 12.3, normalization_90th_ms: 4.2, full_request_90th_ms: 16.5 }
113
+ def data_table_stats(auth_name, action)
106
114
  records = records_for_last_24_hours(auth_name) ||
107
115
  records_for_last_30_days(auth_name) ||
108
116
  records_for_last_12_months(auth_name) ||
109
117
  all_records(auth_name)
110
- stats_calculator_class.new(records).calculate_stats(avg: true, low: true, high: true)
118
+ stats_calculator_class.new(records, action: action).calculate_stats(avg: true, low: true, high: true)
111
119
  end
112
120
 
113
121
  def expected_time_period
@@ -3,16 +3,58 @@ module PrependedLinkedData::FindTerm
3
3
  # Override Qa::Authorities::LinkedData::FindTerm#find method
4
4
  # @return [Hash] single term results in requested format
5
5
  def find(id, language: nil, replacements: {}, subauth: nil, format: nil, jsonld: false, performance_data: false) # rubocop:disable Metrics/ParameterLists
6
+ start_time_s = Time.now.to_f
7
+
6
8
  saved_performance_data = performance_data
7
9
  performance_data = true
8
- full_results = super
9
- return full_results unless full_results.is_a?(Hash) && full_results.key?(:performance)
10
- QaServer::PerformanceHistory.save_result(dt_stamp: Time.now.getlocal,
11
- authority: authority_name,
12
- action: 'fetch',
13
- size_bytes: full_results[:performance][:fetched_bytes],
14
- load_time_ms: (full_results[:performance][:fetch_time_s] * 1000),
15
- normalization_time_ms: (full_results[:performance][:normalization_time_s] * 1000))
16
- saved_performance_data ? full_results : full_results[:results]
10
+ ph_record = QaServer::PerformanceHistory.create_record(authority: authority_name, action: 'fetch')
11
+ @phid = ph_record.id
12
+ begin
13
+ full_results = super
14
+ update_performance_history_record(full_results, start_time_s)
15
+ rescue Exception => e # rubocop:disable Lint/RescueException
16
+ ph_record.destroy
17
+ raise e
18
+ end
19
+ saved_performance_data || !full_results.key?(:results) ? full_results : full_results[:results]
17
20
  end
21
+
22
+ private
23
+
24
+ def update_performance_history_record(full_results, start_time_s)
25
+ ph_record = QaServer::PerformanceHistory.find(@phid)
26
+ return ph_record.destroy unless full_results.is_a?(Hash) && full_results.key?(:performance)
27
+ ph_record.action_time_ms = (Time.now.to_f - start_time_s) * 1000
28
+ ph_record.size_bytes = full_results[:performance][:fetched_bytes]
29
+ ph_record.retrieve_plus_graph_load_time_ms = full_results[:performance][:fetch_time_s] * 1000
30
+ ph_record.normalization_time_ms = full_results[:performance][:normalization_time_s] * 1000
31
+ ph_record.save
32
+ end
33
+
34
+ # Override to append performance history record id into the URL to allow access to the record in RDF::Graph
35
+ def load_graph(url:)
36
+ access_start_dt = Time.now.utc
37
+
38
+ url += "&phid=#{@phid}"
39
+ @full_graph = graph_service.load_graph(url: url)
40
+
41
+ access_end_dt = Time.now.utc
42
+ @access_time_s = access_end_dt - access_start_dt
43
+ @fetched_size = full_graph.triples.to_s.size if performance_data?
44
+ Rails.logger.info("Time to receive data from authority: #{access_time_s}s")
45
+ end
46
+
47
+ # Temporary override to fix bug. Remove when QA PR #273 is merged and a new release is cut
48
+ def normalize_results
49
+ normalize_start_dt = Time.now.utc
50
+
51
+ json = perform_normalization
52
+
53
+ normalize_end_dt = Time.now.utc
54
+ @normalize_time_s = normalize_end_dt - normalize_start_dt
55
+ @normalized_size = json.to_s.size if performance_data?
56
+ Rails.logger.info("Time to convert data to json: #{normalize_time_s}s")
57
+ json = append_performance_data(json) if performance_data?
58
+ json
59
+ end
18
60
  end
@@ -3,16 +3,44 @@ module PrependedLinkedData::SearchQuery
3
3
  # Override Qa::Authorities::LinkedData::SearchQuery#search method
4
4
  # @return [String] json results for search query
5
5
  def search(query, language: nil, replacements: {}, subauth: nil, context: false, performance_data: false) # rubocop:disable Metrics/ParameterLists
6
+ start_time_s = Time.now.to_f
7
+
6
8
  saved_performance_data = performance_data
7
9
  performance_data = true
8
- full_results = super
9
- return full_results unless full_results.is_a?(Hash) && full_results.key?(:performance)
10
- QaServer::PerformanceHistory.save_result(dt_stamp: Time.now.getlocal,
11
- authority: authority_name,
12
- action: 'search',
13
- size_bytes: full_results[:performance][:fetched_bytes],
14
- load_time_ms: (full_results[:performance][:fetch_time_s] * 1000),
15
- normalization_time_ms: (full_results[:performance][:normalization_time_s] * 1000))
16
- saved_performance_data ? full_results : full_results[:results]
10
+ ph_record = QaServer::PerformanceHistory.create_record(authority: authority_name, action: 'search')
11
+ @phid = ph_record.id
12
+ begin
13
+ full_results = super
14
+ update_performance_history_record(full_results, start_time_s)
15
+ rescue Exception => e # rubocop:disable Lint/RescueException
16
+ ph_record.destroy
17
+ raise e
18
+ end
19
+ saved_performance_data || !full_results.key?(:results) ? full_results : full_results[:results]
17
20
  end
21
+
22
+ private
23
+
24
+ def update_performance_history_record(full_results, start_time_s)
25
+ ph_record = QaServer::PerformanceHistory.find(@phid)
26
+ return ph_record.destroy unless full_results.is_a?(Hash) && full_results.key?(:performance)
27
+ ph_record.action_time_ms = (Time.now.to_f - start_time_s) * 1000
28
+ ph_record.size_bytes = full_results[:performance][:fetched_bytes]
29
+ ph_record.retrieve_plus_graph_load_time_ms = full_results[:performance][:fetch_time_s] * 1000
30
+ ph_record.normalization_time_ms = full_results[:performance][:normalization_time_s] * 1000
31
+ ph_record.save
32
+ end
33
+
34
+ # Override to append performance history record id into the URL to allow access to the record in RDF::Graph
35
+ def load_graph(url:)
36
+ access_start_dt = Time.now.utc
37
+
38
+ url += "&phid=#{@phid}"
39
+ @full_graph = graph_service.load_graph(url: url)
40
+
41
+ access_end_dt = Time.now.utc
42
+ @access_time_s = access_end_dt - access_start_dt
43
+ @fetched_size = full_graph.triples.to_s.size if performance_data?
44
+ Rails.logger.info("Time to receive data from authority: #{access_time_s}s")
45
+ end
18
46
  end
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+ require 'rdf/reader'
3
+
4
+ module PrependedRdf::RdfGraph
5
+ ##
6
+ # Loads RDF statements from the given file or URL into `self`.
7
+ #
8
+ # @param [String, #to_s] url
9
+ # @param [Hash{Symbol => Object}] options
10
+ # Options from {RDF::Reader.open}
11
+ # @option options [RDF::Resource] :graph_name
12
+ # Set set graph name of each loaded statement
13
+ # @return [void]
14
+ def load(url, graph_name: nil, **options) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
15
+ raise TypeError, "#{self} is immutable" if immutable?
16
+ phid, real_url = parse_phid(url)
17
+ ph_record = QaServer::PerformanceHistory.find(phid)
18
+ start_time_s = Time.now.to_f
19
+
20
+ reader = RDF::Reader.open(real_url, { base_uri: real_url }.merge(options))
21
+
22
+ end_time_s = Time.now.to_f
23
+ ph_record.retrieve_time_ms = (end_time_s - start_time_s) * 1000
24
+ QaServer.config.performance_tracker.write "#{format('%.6f', end_time_s - start_time_s)}, " # read data
25
+
26
+ start_time_s = Time.now.to_f
27
+
28
+ if graph_name
29
+ statements = []
30
+ reader.each_statement do |statement|
31
+ statement.graph_name = graph_name
32
+ statements << statement
33
+ end
34
+ insert_statements(statements)
35
+ statements.size
36
+ else
37
+ insert_statements(reader)
38
+ nil
39
+ end
40
+
41
+ end_time_s = Time.now.to_f
42
+ ph_record.graph_load_time_ms = (end_time_s - start_time_s) * 1000
43
+ ph_record.save
44
+ QaServer.config.performance_tracker.write "#{format('%.6f', end_time_s - start_time_s)}, " # load graph
45
+ end
46
+
47
+ private
48
+
49
+ def parse_phid(url)
50
+ i = url.rindex('&phid=')
51
+ phid = url[(i + 6)..url.length]
52
+ adjusted_url = url[0..(i - 1)]
53
+ [phid, adjusted_url]
54
+ end
55
+ end
@@ -1,59 +1,84 @@
1
1
  # frozen_string_literal: true
2
2
  # This module provides access methods into the performance data hash.
3
3
  module QaServer::MonitorStatus
4
- module PerformanceDatatableBehavior
4
+ module PerformanceDatatableBehavior # rubocop:disable Metrics/ModuleLength
5
5
  include QaServer::PerformanceHistoryDataKeys
6
6
 
7
- def datatable_stats(authority_data)
8
- authority_data[FOR_DATATABLE]
7
+ def datatable_search_stats(authority_data)
8
+ data_table_for(authority_data, SEARCH)
9
9
  end
10
10
 
11
- def low_load(stats)
12
- format_stat stats[LOW_LOAD]
11
+ def datatable_fetch_stats(authority_data)
12
+ data_table_for(authority_data, FETCH)
13
+ end
14
+
15
+ def datatable_all_actions_stats(authority_data)
16
+ data_table_for(authority_data, ALL_ACTIONS)
17
+ end
18
+
19
+ def low_retrieve(stats)
20
+ format_stat stats, LOW_RETR
21
+ end
22
+
23
+ def low_graph_load(stats)
24
+ format_stat stats, LOW_GRPH
13
25
  end
14
26
 
15
27
  def low_normalization(stats)
16
- format_stat stats[LOW_NORM]
28
+ format_stat stats, LOW_NORM
17
29
  end
18
30
 
19
31
  def low_full_request(stats)
20
- format_stat stats[LOW_FULL]
32
+ format_stat stats, LOW_ACTN
33
+ end
34
+
35
+ def high_retrieve(stats)
36
+ format_stat stats, HIGH_RETR
21
37
  end
22
38
 
23
- def high_load(stats)
24
- format_stat stats[HIGH_LOAD]
39
+ def high_graph_load(stats)
40
+ format_stat stats, HIGH_GRPH
25
41
  end
26
42
 
27
43
  def high_normalization(stats)
28
- format_stat stats[HIGH_NORM]
44
+ format_stat stats, HIGH_NORM
29
45
  end
30
46
 
31
47
  def high_full_request(stats)
32
- format_stat stats[HIGH_FULL]
48
+ format_stat stats, HIGH_ACTN
33
49
  end
34
50
 
35
- def avg_load(stats)
36
- format_stat stats[AVG_LOAD]
51
+ def avg_retrieve(stats)
52
+ format_stat stats, AVG_RETR
53
+ end
54
+
55
+ def avg_graph_load(stats)
56
+ format_stat stats, AVG_GRPH
37
57
  end
38
58
 
39
59
  def avg_normalization(stats)
40
- format_stat stats[AVG_NORM]
60
+ format_stat stats, AVG_NORM
41
61
  end
42
62
 
43
63
  def avg_full_request(stats)
44
- format_stat stats[AVG_FULL]
64
+ format_stat stats, AVG_ACTN
65
+ end
66
+
67
+ def datatable_data_style(stats)
68
+ return "status-not-supported" if unsupported_action?(stats)
69
+ "status-neutral"
45
70
  end
46
71
 
47
72
  def low_full_request_style(stats)
48
- performance_style_class(stats, LOW_FULL)
73
+ performance_style_class(stats, LOW_ACTN)
49
74
  end
50
75
 
51
76
  def high_full_request_style(stats)
52
- performance_style_class(stats, HIGH_FULL)
77
+ performance_style_class(stats, HIGH_ACTN)
53
78
  end
54
79
 
55
80
  def avg_full_request_style(stats)
56
- performance_style_class(stats, AVG_FULL)
81
+ performance_style_class(stats, AVG_ACTN)
57
82
  end
58
83
 
59
84
  def performance_table_description
@@ -75,12 +100,21 @@ module QaServer::MonitorStatus
75
100
  QaServer.config.performance_datatable_default_time_period
76
101
  end
77
102
 
78
- def format_stat(stat)
79
- return '' if stat.nil?
80
- format("%0.1f", stat)
103
+ def data_table_for(authority_data, action)
104
+ authority_data[action][FOR_DATATABLE]
105
+ end
106
+
107
+ def unsupported_action?(stats)
108
+ stats[AVG_ACTN].nan?
109
+ end
110
+
111
+ def format_stat(stats, idx)
112
+ return '' if stats[idx].nil? || unsupported_action?(stats)
113
+ format("%0.1f", stats[idx])
81
114
  end
82
115
 
83
116
  def performance_style_class(stats, stat_key)
117
+ return "status-not-supported" if unsupported_action?(stats)
84
118
  return "status-bad" if max_threshold_exceeded(stats, stat_key)
85
119
  return "status-unknown" if desired_threshold_not_met(stats, stat_key)
86
120
  "status-good"