qa_server 2.2.3 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/app/assets/stylesheets/qa_server/_authorities.scss +4 -1
  4. data/app/assets/stylesheets/qa_server/_check-status.scss +5 -0
  5. data/app/assets/stylesheets/qa_server/_monitor-status.scss +2 -2
  6. data/app/models/concerns/qa_server/performance_history_data_keys.rb +16 -3
  7. data/app/models/qa_server/performance_history.rb +54 -46
  8. data/app/prepends/prepended_linked_data/find_term.rb +51 -9
  9. data/app/prepends/prepended_linked_data/search_query.rb +37 -9
  10. data/app/prepends/prepended_rdf/rdf_graph.rb +55 -0
  11. data/app/presenters/concerns/qa_server/monitor_status/performance_datatable_behavior.rb +55 -21
  12. data/app/presenters/concerns/qa_server/monitor_status/performance_graph_behavior.rb +54 -25
  13. data/app/presenters/qa_server/monitor_status_presenter.rb +11 -8
  14. data/app/services/qa_server/performance_calculator_service.rb +50 -47
  15. data/app/services/qa_server/performance_graph_data_service.rb +13 -12
  16. data/app/services/qa_server/performance_graphing_service.rb +49 -40
  17. data/app/views/qa_server/monitor_status/_performance.html.erb +107 -32
  18. data/config/locales/qa_server.en.yml +9 -2
  19. data/lib/generators/qa_server/config_generator.rb +1 -0
  20. data/lib/generators/qa_server/templates/config/authorities/linked_data/agrovoc_ld4l_cache.json +8 -1
  21. data/lib/generators/qa_server/templates/config/authorities/linked_data/dbpedia_ld4l_cache.json +8 -1
  22. data/lib/generators/qa_server/templates/config/authorities/linked_data/geonames_ld4l_cache.json +11 -1
  23. data/lib/generators/qa_server/templates/config/authorities/linked_data/getty_aat_ld4l_cache.json +8 -1
  24. data/lib/generators/qa_server/templates/config/authorities/linked_data/getty_tgn_ld4l_cache.json +8 -1
  25. data/lib/generators/qa_server/templates/config/authorities/linked_data/getty_ulan_ld4l_cache.json +8 -1
  26. data/lib/generators/qa_server/templates/config/authorities/linked_data/locdemographics_ld4l_cache.json +2 -1
  27. data/lib/generators/qa_server/templates/config/authorities/linked_data/locnames_ld4l_cache.json +51 -42
  28. data/lib/generators/qa_server/templates/config/authorities/linked_data/locnames_rwo_ld4l_cache.json +237 -0
  29. data/lib/generators/qa_server/templates/config/authorities/linked_data/locperformance_ld4l_cache.json +75 -3
  30. data/lib/generators/qa_server/templates/config/authorities/linked_data/mesh_nlm_ld4l_cache.json +8 -1
  31. data/lib/generators/qa_server/templates/config/authorities/linked_data/nalt_ld4l_cache.json +8 -1
  32. data/lib/generators/qa_server/templates/config/authorities/linked_data/oclcfast_ld4l_cache.json +8 -1
  33. data/lib/generators/qa_server/templates/config/authorities/linked_data/rda_registry_ld4l_cache.json +193 -0
  34. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_ld4l_cache_validation.yml +3 -18
  35. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locnames_rwo_ld4l_cache_validation.yml +27 -0
  36. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/locperformance_ld4l_cache_validation.yml +1 -0
  37. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/rda_registry_ld4l_cache_validation.yml +255 -0
  38. data/lib/generators/qa_server/templates/config/authorities/linked_data/scenarios/wikidata_direct_validation.yml +6 -0
  39. data/lib/generators/qa_server/templates/config/authorities/linked_data/wikidata_direct.json +32 -0
  40. data/lib/generators/qa_server/templates/config/initializers/qa_server.rb +14 -5
  41. data/lib/generators/qa_server/templates/db/migrate/20191007134527_update_performance_history_table.rb.erb +19 -0
  42. data/lib/qa_server/configuration.rb +27 -5
  43. data/lib/qa_server/version.rb +1 -1
  44. metadata +10 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: a526113eb2d57af9e670446c76d9c92e33cdb1c8
4
- data.tar.gz: b3f9253e899f8ff605632952f8ef5990a17e9147
3
+ metadata.gz: c54a8298febf924d051837fef371b727c32912f7
4
+ data.tar.gz: 5c038a7a516bd50834670fd393ee215086708b73
5
5
  SHA512:
6
- metadata.gz: d19ffc6e5baf8ebf1d742fb6edc06fc9e620eadd8182a8632653e576fcee0d673b65f50eba97e0c345494b0d2970deb2f174f9583f003bd324934045c888c831
7
- data.tar.gz: afe7fa02263b9b375da202bc87dfdb27d7f790653840a398d28630cfbd3f75d40c944485179260a94dd8a6561fa179a367763607954415cccd9feef86f91958f
6
+ metadata.gz: a36a6e1d5072d3978a18a34b84bf338b9d318c7db8948a6fb8bbb3ae2506714e8281b4dfa42faae61f5410a3379d38c2e2c2077d848c14a8895ce9dfb7193fb8
7
+ data.tar.gz: 1a5df12683d02f30c8bccc9f96d721faaffe06484223fe3ae543dc94f2948c11c164fa93a9833c57b0d523b898de54b19a02b344d5a56f9018d27f2a1b3a8304
data/CHANGELOG.md CHANGED
@@ -1,3 +1,11 @@
1
+ ### 3.0.0 (2019-10-09)
2
+
3
+ * refactor performance data and graphs to include stats by action as well as by authority and time period
4
+ * adds pagination start_record parameter for LD4P cached data
5
+ * adds RDA Registry authority
6
+ * splits LCNAF into 2 authorities for LD4P cached data focused locnames and real world object focused locnames_rwo
7
+ * adds wikidata authority - NOTE: The first pass on fetching through this authority is non-performant.
8
+
1
9
  ### 2.2.3 (2019-09-11)
2
10
 
3
11
  * do not process and store performance data if it wasn't returned in the result
@@ -1,3 +1,6 @@
1
- td.table_subheading {
1
+ th.table_subheading, td.table_subheading {
2
2
  background-color: lightgrey;
3
3
  }
4
+ th.table_subheading {
5
+ text-align: center;
6
+ }
@@ -73,6 +73,11 @@ td.position {
73
73
  background-color: #ffffff;
74
74
  }
75
75
 
76
+ .status-not-supported {
77
+ text-align: center;
78
+ background-color: #F3EEEE;
79
+ }
80
+
76
81
  label.horizontal-list {
77
82
  padding-right: 15px;
78
83
  }
@@ -20,11 +20,11 @@ div.performance-data-section-visible {
20
20
  display: block;
21
21
  }
22
22
 
23
- div.right-menu-section {
23
+ div.performance-graph-menu-section {
24
24
  padding-top: 50px;
25
25
  }
26
26
 
27
- ul.right-menu {
27
+ ul.time-period-menu, ul.action-menu {
28
28
  list-style-type: none;
29
29
  li.clickable {
30
30
  display: inline;
@@ -4,6 +4,10 @@
4
4
  module QaServer
5
5
  module PerformanceHistoryDataKeys
6
6
  ALL_AUTH = :all_authorities
7
+ SEARCH = :search
8
+ FETCH = :fetch
9
+ ALL_ACTIONS = :all_actions
10
+
7
11
  STATS = :stats
8
12
  FOR_DATATABLE = :datatable_stats
9
13
 
@@ -17,13 +21,22 @@ module QaServer
17
21
  BY_MONTH = :month
18
22
 
19
23
  LOW_LOAD = :low_load_ms
24
+ LOW_RETR = :low_retrieve_ms
25
+ LOW_GRPH = :low_load_graph_ms
20
26
  LOW_NORM = :low_normalization_ms
27
+ LOW_ACTN = :low_action_request_ms
21
28
  LOW_FULL = :low_full_request_ms
22
29
  AVG_LOAD = :avg_load_ms
30
+ AVG_RETR = :avg_retrieve_ms
31
+ AVG_GRPH = :avg_load_graph_ms
23
32
  AVG_NORM = :avg_normalization_ms
33
+ AVG_ACTN = :avg_action_request_ms
24
34
  AVG_FULL = :avg_full_request_ms
25
- HIGH_LOAD = :max_load_ms
26
- HIGH_NORM = :max_normalization_ms
27
- HIGH_FULL = :max_full_request_ms
35
+ HIGH_LOAD = :high_load_ms
36
+ HIGH_RETR = :high_retrieve_ms
37
+ HIGH_GRPH = :high_load_graph_ms
38
+ HIGH_NORM = :high_normalization_ms
39
+ HIGH_ACTN = :high_action_request_ms
40
+ HIGH_FULL = :high_full_request_ms
28
41
  end
29
42
  end
@@ -16,15 +16,13 @@ module QaServer
16
16
  include QaServer::PerformanceHistoryDataKeys
17
17
 
18
18
  # Save a scenario result
19
- # @param run_id [Integer] the run on which to gather statistics
20
- # @param result [Hash] the scenario result to be saved
21
- def save_result(dt_stamp:, authority:, action:, size_bytes:, load_time_ms:, normalization_time_ms:) # rubocop:disable Metrics/ParameterLists
22
- create(dt_stamp: dt_stamp,
19
+ # @param authority [String] name of the authority
20
+ # @param action [Symbol] type of action being evaluated (e.g. :fetch, :search)
21
+ # @return ActveRecord::Base for the new performance history record
22
+ def create_record(authority:, action:)
23
+ create(dt_stamp: Time.now.getlocal,
23
24
  authority: authority,
24
- action: action,
25
- size_bytes: size_bytes,
26
- load_time_ms: load_time_ms,
27
- normalization_time_ms: normalization_time_ms)
25
+ action: action)
28
26
  end
29
27
 
30
28
  # Performance data for a day, a month, a year, and all time for each authority.
@@ -32,34 +30,38 @@ module QaServer
32
30
  # @returns [Hash] performance statistics for the past 24 hours
33
31
  # @example
34
32
  # { all_authorities:
35
- # { datatable_stats:
36
- # { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }
37
- # }
38
- # { day:
39
- # { 0: { hour: '1400', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
40
- # 1: { hour: '1500', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
41
- # 2: { hour: '1600', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
42
- # ...,
43
- # 23: { hour: 'NOW', load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
44
- # }
45
- # }
46
- # { month:
47
- # { 0: { day: '07-15-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
48
- # 1: { day: '07-16-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
49
- # 2: { day: '07-17-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
50
- # ...,
51
- # 29: { day: 'TODAY', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
52
- # }
53
- # }
54
- # { year:
55
- # { 0: { month: '09-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
56
- # 1: { month: '10-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
57
- # 2: { month: '11-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
58
- # ...,
59
- # 11: { month: '08-2019', stats: { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
60
- # }
61
- # }
62
- # { AGROVOC_LD4L_CACHE: ... # same data for each authority }
33
+ # { search:
34
+ # {
35
+ # datatable_stats:
36
+ # { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5,
37
+ # retrieve_10th_ms: 12.3, graph_load_10th_ms: 12.3, normalization_10th_ms: 4.2, full_request_10th_ms: 16.5,
38
+ # retrieve_90th_ms: 12.3, graph_load_90th_ms: 12.3, normalization_90th_ms: 4.2, full_request_90th_ms: 16.5 }
39
+ # day:
40
+ # { 0: { hour: '1400', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
41
+ # 1: { hour: '1500', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
42
+ # 2: { hour: '1600', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
43
+ # ...,
44
+ # 23: { hour: 'NOW', retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
45
+ # },
46
+ # month:
47
+ # { 0: { day: '07-15-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
48
+ # 1: { day: '07-16-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
49
+ # 2: { day: '07-17-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
50
+ # ...,
51
+ # 29: { day: 'TODAY', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
52
+ # },
53
+ # year:
54
+ # { 0: { month: '09-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
55
+ # 1: { month: '10-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
56
+ # 2: { month: '11-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }},
57
+ # ...,
58
+ # 11: { month: '08-2019', stats: { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }}
59
+ # }
60
+ # },
61
+ # fetch: { ... # same data as for search_stats }
62
+ # all: { ... # same data as for search_stats }
63
+ # },
64
+ # AGROVOC_LD4L_CACHE: { ... # same data for each authority }
63
65
  # }
64
66
  def performance_data(datatype: :datatable)
65
67
  return if datatype == :none
@@ -87,27 +89,33 @@ module QaServer
87
89
  end
88
90
 
89
91
  def data_for_authority(authority_name: nil, datatype:)
90
- data = {}
91
- data[FOR_DATATABLE] = data_table_stats(authority_name) if calculate_datatable?(datatype)
92
- if calculate_graphdata?(datatype)
93
- data[FOR_DAY] = graph_data_service_class.average_last_24_hours(authority_name)
94
- data[FOR_MONTH] = graph_data_service_class.average_last_30_days(authority_name)
95
- data[FOR_YEAR] = graph_data_service_class.average_last_12_months(authority_name)
92
+ action_data = {}
93
+ [:search, :fetch, :all_actions].each do |action|
94
+ data = {}
95
+ data[FOR_DATATABLE] = data_table_stats(authority_name, action) if calculate_datatable?(datatype)
96
+ if calculate_graphdata?(datatype)
97
+ data[FOR_DAY] = graph_data_service_class.average_last_24_hours(authority_name: authority_name, action: action)
98
+ data[FOR_MONTH] = graph_data_service_class.average_last_30_days(authority_name: authority_name, action: action)
99
+ data[FOR_YEAR] = graph_data_service_class.average_last_12_months(authority_name: authority_name, action: action)
100
+ end
101
+ action_data[action] = data
96
102
  end
97
- data
103
+ action_data
98
104
  end
99
105
 
100
106
  # Get statistics for all available data.
101
107
  # @param [String] auth_name - limit statistics to records for the given authority (default: all authorities)
102
108
  # @returns [Hash] performance statistics for the datatable during the expected time period
103
109
  # @example
104
- # { load_avg_ms: 12.3, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5, etc. }
105
- def data_table_stats(auth_name)
110
+ # { retrieve_avg_ms: 12.3, graph_load_avg_ms: 2.1, normalization_avg_ms: 4.2, full_request_avg_ms: 16.5,
111
+ # retrieve_10th_ms: 12.3, graph_load_10th_ms: 12.3, normalization_10th_ms: 4.2, full_request_10th_ms: 16.5,
112
+ # retrieve_90th_ms: 12.3, graph_load_90th_ms: 12.3, normalization_90th_ms: 4.2, full_request_90th_ms: 16.5 }
113
+ def data_table_stats(auth_name, action)
106
114
  records = records_for_last_24_hours(auth_name) ||
107
115
  records_for_last_30_days(auth_name) ||
108
116
  records_for_last_12_months(auth_name) ||
109
117
  all_records(auth_name)
110
- stats_calculator_class.new(records).calculate_stats(avg: true, low: true, high: true)
118
+ stats_calculator_class.new(records, action: action).calculate_stats(avg: true, low: true, high: true)
111
119
  end
112
120
 
113
121
  def expected_time_period
@@ -3,16 +3,58 @@ module PrependedLinkedData::FindTerm
3
3
  # Override Qa::Authorities::LinkedData::FindTerm#find method
4
4
  # @return [Hash] single term results in requested format
5
5
  def find(id, language: nil, replacements: {}, subauth: nil, format: nil, jsonld: false, performance_data: false) # rubocop:disable Metrics/ParameterLists
6
+ start_time_s = Time.now.to_f
7
+
6
8
  saved_performance_data = performance_data
7
9
  performance_data = true
8
- full_results = super
9
- return full_results unless full_results.is_a?(Hash) && full_results.key?(:performance)
10
- QaServer::PerformanceHistory.save_result(dt_stamp: Time.now.getlocal,
11
- authority: authority_name,
12
- action: 'fetch',
13
- size_bytes: full_results[:performance][:fetched_bytes],
14
- load_time_ms: (full_results[:performance][:fetch_time_s] * 1000),
15
- normalization_time_ms: (full_results[:performance][:normalization_time_s] * 1000))
16
- saved_performance_data ? full_results : full_results[:results]
10
+ ph_record = QaServer::PerformanceHistory.create_record(authority: authority_name, action: 'fetch')
11
+ @phid = ph_record.id
12
+ begin
13
+ full_results = super
14
+ update_performance_history_record(full_results, start_time_s)
15
+ rescue Exception => e # rubocop:disable Lint/RescueException
16
+ ph_record.destroy
17
+ raise e
18
+ end
19
+ saved_performance_data || !full_results.key?(:results) ? full_results : full_results[:results]
17
20
  end
21
+
22
+ private
23
+
24
+ def update_performance_history_record(full_results, start_time_s)
25
+ ph_record = QaServer::PerformanceHistory.find(@phid)
26
+ return ph_record.destroy unless full_results.is_a?(Hash) && full_results.key?(:performance)
27
+ ph_record.action_time_ms = (Time.now.to_f - start_time_s) * 1000
28
+ ph_record.size_bytes = full_results[:performance][:fetched_bytes]
29
+ ph_record.retrieve_plus_graph_load_time_ms = full_results[:performance][:fetch_time_s] * 1000
30
+ ph_record.normalization_time_ms = full_results[:performance][:normalization_time_s] * 1000
31
+ ph_record.save
32
+ end
33
+
34
+ # Override to append performance history record id into the URL to allow access to the record in RDF::Graph
35
+ def load_graph(url:)
36
+ access_start_dt = Time.now.utc
37
+
38
+ url += "&phid=#{@phid}"
39
+ @full_graph = graph_service.load_graph(url: url)
40
+
41
+ access_end_dt = Time.now.utc
42
+ @access_time_s = access_end_dt - access_start_dt
43
+ @fetched_size = full_graph.triples.to_s.size if performance_data?
44
+ Rails.logger.info("Time to receive data from authority: #{access_time_s}s")
45
+ end
46
+
47
+ # Temporary override to fix bug. Remove when QA PR #273 is merged and a new release is cut
48
+ def normalize_results
49
+ normalize_start_dt = Time.now.utc
50
+
51
+ json = perform_normalization
52
+
53
+ normalize_end_dt = Time.now.utc
54
+ @normalize_time_s = normalize_end_dt - normalize_start_dt
55
+ @normalized_size = json.to_s.size if performance_data?
56
+ Rails.logger.info("Time to convert data to json: #{normalize_time_s}s")
57
+ json = append_performance_data(json) if performance_data?
58
+ json
59
+ end
18
60
  end
@@ -3,16 +3,44 @@ module PrependedLinkedData::SearchQuery
3
3
  # Override Qa::Authorities::LinkedData::SearchQuery#search method
4
4
  # @return [String] json results for search query
5
5
  def search(query, language: nil, replacements: {}, subauth: nil, context: false, performance_data: false) # rubocop:disable Metrics/ParameterLists
6
+ start_time_s = Time.now.to_f
7
+
6
8
  saved_performance_data = performance_data
7
9
  performance_data = true
8
- full_results = super
9
- return full_results unless full_results.is_a?(Hash) && full_results.key?(:performance)
10
- QaServer::PerformanceHistory.save_result(dt_stamp: Time.now.getlocal,
11
- authority: authority_name,
12
- action: 'search',
13
- size_bytes: full_results[:performance][:fetched_bytes],
14
- load_time_ms: (full_results[:performance][:fetch_time_s] * 1000),
15
- normalization_time_ms: (full_results[:performance][:normalization_time_s] * 1000))
16
- saved_performance_data ? full_results : full_results[:results]
10
+ ph_record = QaServer::PerformanceHistory.create_record(authority: authority_name, action: 'search')
11
+ @phid = ph_record.id
12
+ begin
13
+ full_results = super
14
+ update_performance_history_record(full_results, start_time_s)
15
+ rescue Exception => e # rubocop:disable Lint/RescueException
16
+ ph_record.destroy
17
+ raise e
18
+ end
19
+ saved_performance_data || !full_results.key?(:results) ? full_results : full_results[:results]
17
20
  end
21
+
22
+ private
23
+
24
+ def update_performance_history_record(full_results, start_time_s)
25
+ ph_record = QaServer::PerformanceHistory.find(@phid)
26
+ return ph_record.destroy unless full_results.is_a?(Hash) && full_results.key?(:performance)
27
+ ph_record.action_time_ms = (Time.now.to_f - start_time_s) * 1000
28
+ ph_record.size_bytes = full_results[:performance][:fetched_bytes]
29
+ ph_record.retrieve_plus_graph_load_time_ms = full_results[:performance][:fetch_time_s] * 1000
30
+ ph_record.normalization_time_ms = full_results[:performance][:normalization_time_s] * 1000
31
+ ph_record.save
32
+ end
33
+
34
+ # Override to append performance history record id into the URL to allow access to the record in RDF::Graph
35
+ def load_graph(url:)
36
+ access_start_dt = Time.now.utc
37
+
38
+ url += "&phid=#{@phid}"
39
+ @full_graph = graph_service.load_graph(url: url)
40
+
41
+ access_end_dt = Time.now.utc
42
+ @access_time_s = access_end_dt - access_start_dt
43
+ @fetched_size = full_graph.triples.to_s.size if performance_data?
44
+ Rails.logger.info("Time to receive data from authority: #{access_time_s}s")
45
+ end
18
46
  end
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+ require 'rdf/reader'
3
+
4
+ module PrependedRdf::RdfGraph
5
+ ##
6
+ # Loads RDF statements from the given file or URL into `self`.
7
+ #
8
+ # @param [String, #to_s] url
9
+ # @param [Hash{Symbol => Object}] options
10
+ # Options from {RDF::Reader.open}
11
+ # @option options [RDF::Resource] :graph_name
12
+ # Set set graph name of each loaded statement
13
+ # @return [void]
14
+ def load(url, graph_name: nil, **options) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
15
+ raise TypeError, "#{self} is immutable" if immutable?
16
+ phid, real_url = parse_phid(url)
17
+ ph_record = QaServer::PerformanceHistory.find(phid)
18
+ start_time_s = Time.now.to_f
19
+
20
+ reader = RDF::Reader.open(real_url, { base_uri: real_url }.merge(options))
21
+
22
+ end_time_s = Time.now.to_f
23
+ ph_record.retrieve_time_ms = (end_time_s - start_time_s) * 1000
24
+ QaServer.config.performance_tracker.write "#{format('%.6f', end_time_s - start_time_s)}, " # read data
25
+
26
+ start_time_s = Time.now.to_f
27
+
28
+ if graph_name
29
+ statements = []
30
+ reader.each_statement do |statement|
31
+ statement.graph_name = graph_name
32
+ statements << statement
33
+ end
34
+ insert_statements(statements)
35
+ statements.size
36
+ else
37
+ insert_statements(reader)
38
+ nil
39
+ end
40
+
41
+ end_time_s = Time.now.to_f
42
+ ph_record.graph_load_time_ms = (end_time_s - start_time_s) * 1000
43
+ ph_record.save
44
+ QaServer.config.performance_tracker.write "#{format('%.6f', end_time_s - start_time_s)}, " # load graph
45
+ end
46
+
47
+ private
48
+
49
+ def parse_phid(url)
50
+ i = url.rindex('&phid=')
51
+ phid = url[(i + 6)..url.length]
52
+ adjusted_url = url[0..(i - 1)]
53
+ [phid, adjusted_url]
54
+ end
55
+ end
@@ -1,59 +1,84 @@
1
1
  # frozen_string_literal: true
2
2
  # This module provides access methods into the performance data hash.
3
3
  module QaServer::MonitorStatus
4
- module PerformanceDatatableBehavior
4
+ module PerformanceDatatableBehavior # rubocop:disable Metrics/ModuleLength
5
5
  include QaServer::PerformanceHistoryDataKeys
6
6
 
7
- def datatable_stats(authority_data)
8
- authority_data[FOR_DATATABLE]
7
+ def datatable_search_stats(authority_data)
8
+ data_table_for(authority_data, SEARCH)
9
9
  end
10
10
 
11
- def low_load(stats)
12
- format_stat stats[LOW_LOAD]
11
+ def datatable_fetch_stats(authority_data)
12
+ data_table_for(authority_data, FETCH)
13
+ end
14
+
15
+ def datatable_all_actions_stats(authority_data)
16
+ data_table_for(authority_data, ALL_ACTIONS)
17
+ end
18
+
19
+ def low_retrieve(stats)
20
+ format_stat stats, LOW_RETR
21
+ end
22
+
23
+ def low_graph_load(stats)
24
+ format_stat stats, LOW_GRPH
13
25
  end
14
26
 
15
27
  def low_normalization(stats)
16
- format_stat stats[LOW_NORM]
28
+ format_stat stats, LOW_NORM
17
29
  end
18
30
 
19
31
  def low_full_request(stats)
20
- format_stat stats[LOW_FULL]
32
+ format_stat stats, LOW_ACTN
33
+ end
34
+
35
+ def high_retrieve(stats)
36
+ format_stat stats, HIGH_RETR
21
37
  end
22
38
 
23
- def high_load(stats)
24
- format_stat stats[HIGH_LOAD]
39
+ def high_graph_load(stats)
40
+ format_stat stats, HIGH_GRPH
25
41
  end
26
42
 
27
43
  def high_normalization(stats)
28
- format_stat stats[HIGH_NORM]
44
+ format_stat stats, HIGH_NORM
29
45
  end
30
46
 
31
47
  def high_full_request(stats)
32
- format_stat stats[HIGH_FULL]
48
+ format_stat stats, HIGH_ACTN
33
49
  end
34
50
 
35
- def avg_load(stats)
36
- format_stat stats[AVG_LOAD]
51
+ def avg_retrieve(stats)
52
+ format_stat stats, AVG_RETR
53
+ end
54
+
55
+ def avg_graph_load(stats)
56
+ format_stat stats, AVG_GRPH
37
57
  end
38
58
 
39
59
  def avg_normalization(stats)
40
- format_stat stats[AVG_NORM]
60
+ format_stat stats, AVG_NORM
41
61
  end
42
62
 
43
63
  def avg_full_request(stats)
44
- format_stat stats[AVG_FULL]
64
+ format_stat stats, AVG_ACTN
65
+ end
66
+
67
+ def datatable_data_style(stats)
68
+ return "status-not-supported" if unsupported_action?(stats)
69
+ "status-neutral"
45
70
  end
46
71
 
47
72
  def low_full_request_style(stats)
48
- performance_style_class(stats, LOW_FULL)
73
+ performance_style_class(stats, LOW_ACTN)
49
74
  end
50
75
 
51
76
  def high_full_request_style(stats)
52
- performance_style_class(stats, HIGH_FULL)
77
+ performance_style_class(stats, HIGH_ACTN)
53
78
  end
54
79
 
55
80
  def avg_full_request_style(stats)
56
- performance_style_class(stats, AVG_FULL)
81
+ performance_style_class(stats, AVG_ACTN)
57
82
  end
58
83
 
59
84
  def performance_table_description
@@ -75,12 +100,21 @@ module QaServer::MonitorStatus
75
100
  QaServer.config.performance_datatable_default_time_period
76
101
  end
77
102
 
78
- def format_stat(stat)
79
- return '' if stat.nil?
80
- format("%0.1f", stat)
103
+ def data_table_for(authority_data, action)
104
+ authority_data[action][FOR_DATATABLE]
105
+ end
106
+
107
+ def unsupported_action?(stats)
108
+ stats[AVG_ACTN].nan?
109
+ end
110
+
111
+ def format_stat(stats, idx)
112
+ return '' if stats[idx].nil? || unsupported_action?(stats)
113
+ format("%0.1f", stats[idx])
81
114
  end
82
115
 
83
116
  def performance_style_class(stats, stat_key)
117
+ return "status-not-supported" if unsupported_action?(stats)
84
118
  return "status-bad" if max_threshold_exceeded(stats, stat_key)
85
119
  return "status-unknown" if desired_threshold_not_met(stats, stat_key)
86
120
  "status-good"