qa 4.0.0 → 4.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/app/controllers/qa/linked_data_terms_controller.rb +30 -9
  4. data/app/controllers/qa/terms_controller.rb +3 -2
  5. data/app/models/qa/linked_data/config/context_property_map.rb +6 -25
  6. data/app/services/qa/iri_template_service.rb +32 -24
  7. data/app/services/qa/linked_data/authority_service.rb +8 -0
  8. data/app/services/qa/linked_data/authority_url_service.rb +27 -8
  9. data/app/services/qa/linked_data/deep_sort_service.rb +3 -2
  10. data/app/services/qa/linked_data/graph_service.rb +13 -0
  11. data/app/services/qa/linked_data/language_service.rb +12 -0
  12. data/app/services/qa/linked_data/language_sort_service.rb +7 -2
  13. data/app/services/qa/linked_data/ldpath_service.rb +40 -0
  14. data/app/services/qa/linked_data/mapper/graph_ldpath_mapper_service.rb +49 -0
  15. data/app/services/qa/linked_data/mapper/graph_mapper_service.rb +3 -11
  16. data/app/services/qa/linked_data/mapper/graph_predicate_mapper_service.rb +40 -0
  17. data/app/services/qa/linked_data/mapper/search_results_mapper_service.rb +58 -11
  18. data/app/services/qa/linked_data/mapper/term_results_mapper_service.rb +80 -0
  19. data/config/authorities/linked_data/loc.json +13 -7
  20. data/config/authorities/linked_data/oclc_fast.json +13 -8
  21. data/lib/generators/qa/discogs/USAGE +10 -0
  22. data/lib/generators/qa/discogs/discogs_generator.rb +12 -0
  23. data/lib/generators/qa/discogs/templates/config/discogs-formats.yml +346 -0
  24. data/lib/generators/qa/discogs/templates/config/discogs-genres.yml +627 -0
  25. data/lib/generators/qa/install/templates/config/initializers/qa.rb +4 -0
  26. data/lib/qa.rb +6 -0
  27. data/lib/qa/authorities.rb +2 -0
  28. data/lib/qa/authorities/discogs.rb +28 -0
  29. data/lib/qa/authorities/discogs/discogs_instance_builder.rb +145 -0
  30. data/lib/qa/authorities/discogs/discogs_translation.rb +126 -0
  31. data/lib/qa/authorities/discogs/discogs_utils.rb +89 -0
  32. data/lib/qa/authorities/discogs/discogs_works_builder.rb +153 -0
  33. data/lib/qa/authorities/discogs/generic_authority.rb +151 -0
  34. data/lib/qa/authorities/discogs_subauthority.rb +9 -0
  35. data/lib/qa/authorities/linked_data/config.rb +7 -3
  36. data/lib/qa/authorities/linked_data/config/search_config.rb +99 -11
  37. data/lib/qa/authorities/linked_data/config/term_config.rb +112 -8
  38. data/lib/qa/authorities/linked_data/find_term.rb +154 -84
  39. data/lib/qa/authorities/linked_data/search_query.rb +76 -13
  40. data/lib/qa/configuration.rb +8 -0
  41. data/lib/qa/version.rb +1 -1
  42. data/spec/controllers/linked_data_terms_controller_spec.rb +151 -30
  43. data/spec/controllers/terms_controller_spec.rb +4 -0
  44. data/spec/features/linked_data/language_spec.rb +298 -0
  45. data/spec/fixtures/authorities/linked_data/lod_full_config.json +21 -5
  46. data/spec/fixtures/authorities/linked_data/lod_lang_defaults.json +4 -4
  47. data/spec/fixtures/authorities/linked_data/lod_lang_multi_defaults.json +4 -4
  48. data/spec/fixtures/authorities/linked_data/lod_lang_no_defaults.json +4 -5
  49. data/spec/fixtures/authorities/linked_data/lod_lang_param.json +4 -4
  50. data/spec/fixtures/authorities/linked_data/lod_term_uri_param_config.json +1 -1
  51. data/spec/fixtures/discogs-find-response-json.json +1 -0
  52. data/spec/fixtures/discogs-find-response-jsonld-master.json +1 -0
  53. data/spec/fixtures/discogs-find-response-jsonld-release.json +1 -0
  54. data/spec/fixtures/discogs-id-matches-master.json +1 -0
  55. data/spec/fixtures/discogs-id-matches-release.json +1 -0
  56. data/spec/fixtures/discogs-id-not-found-master.json +1 -0
  57. data/spec/fixtures/discogs-id-not-found-release.json +1 -0
  58. data/spec/fixtures/discogs-search-response-no-auth.json +1 -0
  59. data/spec/fixtures/discogs-search-response-no-subauth.json +1 -0
  60. data/spec/fixtures/discogs-search-response-subauth.json +1 -0
  61. data/spec/fixtures/lod_lang_search_enesfrde.rdf.xml +60 -0
  62. data/spec/fixtures/lod_lang_search_sv.rdf.xml +42 -0
  63. data/spec/fixtures/lod_loc_term_found.rdf.xml +5 -0
  64. data/spec/lib/authorities/discogs/generic_authority_spec.rb +235 -0
  65. data/spec/lib/authorities/discogs_spec.rb +17 -0
  66. data/spec/lib/authorities/linked_data/config_spec.rb +68 -5
  67. data/spec/lib/authorities/linked_data/find_term_spec.rb +298 -3
  68. data/spec/lib/authorities/linked_data/generic_authority_spec.rb +46 -485
  69. data/spec/lib/authorities/linked_data/search_config_spec.rb +154 -3
  70. data/spec/lib/authorities/linked_data/search_query_spec.rb +240 -3
  71. data/spec/lib/authorities/linked_data/term_config_spec.rb +193 -5
  72. data/spec/lib/configuration_spec.rb +18 -0
  73. data/spec/models/linked_data/config/context_property_map_spec.rb +3 -31
  74. data/spec/services/iri_template_service_spec.rb +54 -12
  75. data/spec/{lib/authorities → services}/linked_data/authority_service_spec.rb +47 -0
  76. data/spec/services/linked_data/language_service_spec.rb +52 -11
  77. data/spec/services/linked_data/ldpath_service_spec.rb +61 -0
  78. data/spec/services/linked_data/mapper/graph_ldpath_mapper_service_spec.rb +118 -0
  79. data/spec/services/linked_data/mapper/graph_predicate_mapper_service_spec.rb +110 -0
  80. data/spec/services/linked_data/mapper/term_results_mapper_service_spec.rb +94 -0
  81. data/spec/spec_helper.rb +1 -1
  82. data/spec/support/matchers/include_hash.rb +5 -0
  83. data/spec/test_app_templates/lib/generators/test_app_generator.rb +4 -0
  84. metadata +73 -5
  85. data/lib/qa/authorities/linked_data/rdf_helper.rb +0 -49
@@ -3,24 +3,29 @@
3
3
  module Qa::Authorities
4
4
  module LinkedData
5
5
  class FindTerm
6
- include Qa::Authorities::LinkedData::RdfHelper
6
+ class_attribute :authority_service, :graph_service, :language_service, :language_sort_service, :results_mapper_service
7
+ self.authority_service = Qa::LinkedData::AuthorityUrlService
8
+ self.graph_service = Qa::LinkedData::GraphService
9
+ self.language_service = Qa::LinkedData::LanguageService
10
+ self.language_sort_service = Qa::LinkedData::LanguageSortService
11
+ self.results_mapper_service = Qa::LinkedData::Mapper::TermResultsMapperService
7
12
 
8
13
  # @param [TermConfig] term_config The term portion of the config
9
14
  def initialize(term_config)
10
15
  @term_config = term_config
11
16
  end
12
17
 
13
- attr_reader :term_config, :full_graph, :filtered_graph, :language
14
- private :full_graph, :filtered_graph, :language
18
+ attr_reader :term_config, :full_graph, :filtered_graph, :language, :id, :access_time_s, :normalize_time_s
19
+ private :full_graph, :filtered_graph, :language, :id, :access_time_s, :normalize_time_s
15
20
 
16
- delegate :term_subauthority?, to: :term_config
21
+ delegate :term_subauthority?, :prefixes, :authority_name, to: :term_config
17
22
 
18
23
  # Find a single term in a linked data authority
19
24
  # @param [String] the id of the term to fetch
20
25
  # @param [Symbol] (optional) language: language used to select literals when multi-language is supported (e.g. :en, :fr, etc.)
21
26
  # @param [Hash] (optional) replacements: replacement values with { pattern_name (defined in YAML config) => value }
22
27
  # @param [String] subauth: the subauthority from which to fetch the term
23
- # @return [String] json results
28
+ # @return [Hash] json results
24
29
  # @example Json Results for Linked Data Term
25
30
  # { "uri":"http://id.worldcat.org/fast/530369",
26
31
  # "id":"530369","label":"Cornell University",
@@ -34,102 +39,159 @@ module Qa::Authorities
34
39
  # "http://schema.org/name":["Cornell University","Ithaca (N.Y.). Cornell University"],
35
40
  # "http://www.w3.org/2004/02/skos/core#altLabel":["Ithaca (N.Y.). Cornell University"],
36
41
  # "http://schema.org/sameAs":["http://id.loc.gov/authorities/names/n79021621","https://viaf.org/viaf/126293486"] } }
37
- def find(id, language: nil, replacements: {}, subauth: nil, jsonld: false)
42
+ def find(id, language: nil, replacements: {}, subauth: nil, jsonld: false, performance_data: false) # rubocop:disable Metrics/ParameterLists
38
43
  raise Qa::InvalidLinkedDataAuthority, "Unable to initialize linked data term sub-authority #{subauth}" unless subauth.nil? || term_subauthority?(subauth)
39
- @language = Qa::LinkedData::LanguageService.preferred_language(user_language: language, authority_language: term_config.term_language)
40
- url = Qa::LinkedData::AuthorityUrlService.build_url(action_config: term_config, action: :term, action_request: id, substitutions: replacements, subauthority: subauth)
44
+ @language = language_service.preferred_language(user_language: language, authority_language: term_config.term_language)
45
+ @id = id
46
+ @performance_data = performance_data
47
+ @jsonld = jsonld
48
+ url = authority_service.build_url(action_config: term_config, action: :term, action_request: normalize_id, substitutions: replacements, subauthority: subauth, language: @language)
41
49
  Rails.logger.info "QA Linked Data term url: #{url}"
42
50
  load_graph(url: url)
43
- return "{}" unless full_graph.size.positive?
44
- return full_graph.dump(:jsonld, standard_prefixes: true) if jsonld
45
- parse_term_authority_response(id)
51
+ normalize_results
46
52
  end
47
53
 
48
54
  private
49
55
 
50
56
  def load_graph(url:)
51
- # @graph = Qa::LinkedData::GraphService.load_graph(url: url)
52
- @full_graph = Qa::LinkedData::GraphService.load_graph(url: url)
53
- return unless @full_graph.size.positive?
54
- @filtered_graph = Qa::LinkedData::GraphService.deep_copy(graph: @full_graph)
55
- @filtered_graph = Qa::LinkedData::GraphService.filter(graph: @filtered_graph, language: language) unless language.blank?
57
+ access_start_dt = Time.now.utc
58
+
59
+ @full_graph = graph_service.load_graph(url: url)
60
+
61
+ access_end_dt = Time.now.utc
62
+ @access_time_s = access_end_dt - access_start_dt
63
+ Rails.logger.info("Time to receive data from authority: #{access_time_s}s")
56
64
  end
57
65
 
58
- def parse_term_authority_response(id)
59
- results = extract_preds(filtered_graph, preds_for_term)
60
- consolidated_results = consolidate_term_results(results)
61
- json_results = convert_term_to_json(consolidated_results)
62
- termhash = select_json_result_for_id(json_results, id)
63
- predicates_hash = predicates_with_subject_uri(termhash[:uri])
64
- termhash['predicates'] = predicates_hash unless predicates_hash.length <= 0
65
- termhash
66
+ def normalize_results
67
+ normalize_start_dt = Time.now.utc
68
+ normalize_end_dt = Time.now.utc
69
+
70
+ json = perform_normalization
71
+
72
+ @normalize_time_s = normalize_end_dt - normalize_start_dt
73
+ Rails.logger.info("Time to convert data to json: #{normalize_time_s}s")
74
+ json = append_performance_data(json) if performance_data? && !jsonld?
75
+ json
66
76
  end
67
77
 
68
- def preds_for_term
69
- { required: required_term_preds, optional: optional_term_preds }
78
+ def perform_normalization
79
+ return "{}" unless full_graph.size.positive?
80
+ return full_graph.dump(:jsonld, standard_prefixes: true) if jsonld?
81
+
82
+ filter_graph
83
+ results = map_results
84
+ convert_results_to_json(results)
70
85
  end
71
86
 
72
- def required_term_preds
73
- label_pred_uri = term_config.term_results_label_predicate
74
- raise Qa::InvalidConfiguration, "required label_predicate is missing in configuration for LOD authority #{auth_name}" if label_pred_uri.nil?
75
- { label: label_pred_uri }
76
- end
77
-
78
- def optional_term_preds
79
- preds = {}
80
- preds[:altlabel] = term_config.term_results_altlabel_predicate unless term_config.term_results_altlabel_predicate.nil?
81
- preds[:id] = term_config.term_results_id_predicate unless term_config.term_results_id_predicate.nil?
82
- preds[:narrower] = term_config.term_results_narrower_predicate unless term_config.term_results_narrower_predicate.nil?
83
- preds[:broader] = term_config.term_results_broader_predicate unless term_config.term_results_broader_predicate.nil?
84
- preds[:sameas] = term_config.term_results_sameas_predicate unless term_config.term_results_sameas_predicate.nil?
85
- preds
86
- end
87
-
88
- def consolidate_term_results(results) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize # TODO: Explore a way to simplify
89
- consolidated_results = {}
90
- results.each do |statement|
91
- stmt_hash = statement.to_h
92
- uri = stmt_hash[:uri].to_s
93
- consolidated_hash = init_consolidated_hash(consolidated_results, uri, stmt_hash[:id].to_s)
94
-
95
- consolidated_hash[:label] = object_value(stmt_hash, consolidated_hash, :label, false)
96
- altlabel = object_value(stmt_hash, consolidated_hash, :altlabel, false)
97
- narrower = object_value(stmt_hash, consolidated_hash, :narrower)
98
- broader = object_value(stmt_hash, consolidated_hash, :broader)
99
- sameas = object_value(stmt_hash, consolidated_hash, :sameas)
100
-
101
- consolidated_hash[:altlabel] = altlabel unless altlabel.nil?
102
- consolidated_hash[:narrower] = narrower unless narrower.nil?
103
- consolidated_hash[:broader] = broader unless broader.nil?
104
- consolidated_hash[:sameas] = sameas unless sameas.nil?
105
- consolidated_results[uri] = consolidated_hash
106
- end
107
- consolidated_results.each do |res|
108
- consolidated_hash = res[1]
109
- consolidated_hash[:label] = sort_string_by_language consolidated_hash[:label]
110
- consolidated_hash[:altlabel] = sort_string_by_language consolidated_hash[:altlabel]
111
- consolidated_hash[:sort] = sort_string_by_language consolidated_hash[:sort]
112
- end
113
- consolidated_results
114
- end
115
-
116
- def convert_term_to_json(consolidated_results)
117
- json_results = []
118
- consolidated_results.each do |uri, h|
119
- json_hash = { uri: uri, id: h[:id], label: h[:label] }
120
- json_hash[:altlabel] = h[:altlabel] unless h[:altlabel].nil?
121
- json_hash[:narrower] = h[:narrower] unless h[:narrower].nil?
122
- json_hash[:broader] = h[:broader] unless h[:broader].nil?
123
- json_hash[:sameas] = h[:sameas] unless h[:sameas].nil?
124
- json_results << json_hash
87
+ def filter_graph
88
+ @filtered_graph = graph_service.deep_copy(graph: @full_graph)
89
+ @filtered_graph = graph_service.filter(graph: @filtered_graph, language: language) unless language.blank?
90
+ end
91
+
92
+ def map_results
93
+ predicate_map = preds_for_term
94
+ ldpath_map = ldpaths_for_term
95
+
96
+ raise Qa::InvalidConfiguration, "do not specify results using both predicates and ldpath in term configuration for LOD authority #{authority_name} (ldpath is preferred)" if predicate_map.present? && ldpath_map.present? # rubocop:disable Metrics/LineLength
97
+ raise Qa::InvalidConfiguration, "must specify label_ldpath or label_predicate in term configuration for LOD authority #{authority_name} (label_ldpath is preferred)" unless ldpath_map.key?(:label) || predicate_map.key?(:label) # rubocop:disable Metrics/LineLength
98
+
99
+ if predicate_map.present?
100
+ Qa.deprecation_warning(
101
+ in_msg: 'Qa::Authorities::LinkedData::FindTerm',
102
+ msg: 'defining results using predicates in term config is deprecated; update to define using ldpaths'
103
+ )
125
104
  end
126
- json_results
105
+
106
+ results_mapper_service.map_values(graph: @filtered_graph, subject_uri: uri, prefixes: prefixes,
107
+ ldpath_map: ldpaths_for_term, predicate_map: preds_for_term)
108
+ end
109
+
110
+ def normalize_id
111
+ return id if expects_uri?
112
+ authority_name.to_s.casecmp('loc').zero? ? id.delete(' ') : id
113
+ end
114
+
115
+ def expects_uri?
116
+ term_config.term_id_expects_uri?
117
+ end
118
+
119
+ def uri
120
+ return @uri if @uri.present?
121
+ return @uri = RDF::URI.new(id) if expects_uri?
122
+ @uri = graph_service.subjects_for_object_value(graph: @filtered_graph, predicate: RDF::URI.new(term_config.term_results_id_predicate), object_value: id.gsub('%20', ' ')).first
123
+ end
124
+
125
+ def ldpaths_for_term
126
+ label_ldpath = term_config.term_results_label_ldpath
127
+ return {} if label_ldpath.blank?
128
+ ldpaths = { label: label_ldpath }
129
+ ldpaths.merge(optional_ldpaths)
130
+ end
131
+
132
+ def optional_ldpaths
133
+ opt_ldpaths = {}
134
+ opt_ldpaths[:altlabel] = term_config.term_results_altlabel_ldpath
135
+ opt_ldpaths[:id] = term_config.term_results_id_ldpath
136
+ opt_ldpaths[:narrower] = term_config.term_results_narrower_ldpath
137
+ opt_ldpaths[:broader] = term_config.term_results_broader_ldpath
138
+ opt_ldpaths[:sameas] = term_config.term_results_sameas_ldpath
139
+ opt_ldpaths.delete_if { |_k, v| v.blank? }
140
+ end
141
+
142
+ def jsonld?
143
+ @jsonld == true
144
+ end
145
+
146
+ def performance_data?
147
+ @performance_data == true
148
+ end
149
+
150
+ def preds_for_term
151
+ label_pred_uri = term_config.term_results_label_predicate
152
+ return {} if label_pred_uri.blank?
153
+ preds = { label: label_pred_uri }
154
+ preds.merge(optional_preds)
127
155
  end
128
156
 
129
- def select_json_result_for_id(json_results, id)
130
- json_results.select! { |r| r[:uri].include? id } if json_results.size > 1
131
- json_results.select! { |r| r[:uri].ends_with? id } if json_results.size > 1
132
- json_results.first
157
+ def optional_preds
158
+ opt_preds = {}
159
+ opt_preds[:altlabel] = term_config.term_results_altlabel_predicate
160
+ opt_preds[:id] = term_config.term_results_id_predicate
161
+ opt_preds[:narrower] = term_config.term_results_narrower_predicate
162
+ opt_preds[:broader] = term_config.term_results_broader_predicate
163
+ opt_preds[:sameas] = term_config.term_results_sameas_predicate
164
+ opt_preds.delete_if { |_k, v| v.blank? }
165
+ end
166
+
167
+ def convert_results_to_json(results)
168
+ json_hash = { uri: uri.to_s }
169
+ json_hash[:id] = results.key?(:id) && results[:id].present? ? results[:id].first.to_s : uri.to_s
170
+ json_hash[:label] = sort_literals(results, :label)
171
+ json_hash.merge!(optional_results_to_json(results))
172
+ predicates_hash = predicates_with_subject_uri(uri)
173
+ json_hash['predicates'] = predicates_hash if predicates_hash.present?
174
+ json_hash
175
+ end
176
+
177
+ def optional_results_to_json(results)
178
+ opt_results_json = {}
179
+ opt_results_json[:altlabel] = sort_literals(results, :altlabel)
180
+ opt_results_json[:narrower] = extract_result(results, :narrower)
181
+ opt_results_json[:broader] = extract_result(results, :broader)
182
+ opt_results_json[:sameas] = extract_result(results, :sameas)
183
+ opt_results_json.delete_if { |_k, v| v.blank? }
184
+ end
185
+
186
+ def extract_result(results, key)
187
+ return nil unless results.key?(key) && results[key].present?
188
+ results[key].map(&:to_s)
189
+ end
190
+
191
+ def sort_literals(results, key)
192
+ return nil unless results.key? key
193
+ return [] if results[key].blank?
194
+ language_sort_service.new(results[key], language).uniq_sorted_strings
133
195
  end
134
196
 
135
197
  def predicates_with_subject_uri(expected_uri) # rubocop:disable Metrics/MethodLength
@@ -152,6 +214,14 @@ module Qa::Authorities
152
214
  end
153
215
  predicates_hash
154
216
  end
217
+
218
+ def append_performance_data(results)
219
+ performance = { predicate_count: results['predicates'].size,
220
+ fetch_time_s: access_time_s,
221
+ normalization_time_s: normalize_time_s,
222
+ total_time_s: (access_time_s + normalize_time_s) }
223
+ { performance: performance, results: results }
224
+ end
155
225
  end
156
226
  end
157
227
  end
@@ -15,10 +15,10 @@ module Qa::Authorities
15
15
  @search_config = search_config
16
16
  end
17
17
 
18
- attr_reader :search_config, :graph, :language
19
- private :graph, :language
18
+ attr_reader :search_config, :graph, :language, :access_time_s, :normalize_time_s
19
+ private :graph, :language, :access_time_s, :normalize_time_s
20
20
 
21
- delegate :subauthority?, :supports_sort?, to: :search_config
21
+ delegate :subauthority?, :supports_sort?, :prefixes, :authority_name, to: :search_config
22
22
 
23
23
  # Search a linked data authority
24
24
  # @praram [String] the query
@@ -26,32 +26,66 @@ module Qa::Authorities
26
26
  # @param replacements [Hash] (optional) replacement values with { pattern_name (defined in YAML config) => value }
27
27
  # @param subauth [String] (optional) the subauthority to query
28
28
  # @param context [Boolean] (optional) true if context should be returned with the results; otherwise, false (default: false)
29
+ # @param performance_data [Boolean] (optional) true if include_performance_data should be returned with the results; otherwise, false (default: false)
29
30
  # @return [String] json results
30
31
  # @example Json Results for Linked Data Search
31
32
  # [ {"uri":"http://id.worldcat.org/fast/5140","id":"5140","label":"Cornell, Joseph"},
32
33
  # {"uri":"http://id.worldcat.org/fast/72456","id":"72456","label":"Cornell, Sarah Maria, 1802-1832"},
33
34
  # {"uri":"http://id.worldcat.org/fast/409667","id":"409667","label":"Cornell, Ezra, 1807-1874"} ]
34
- def search(query, language: nil, replacements: {}, subauth: nil, context: false)
35
+ def search(query, language: nil, replacements: {}, subauth: nil, context: false, performance_data: false) # rubocop:disable Metrics/ParameterLists
35
36
  raise Qa::InvalidLinkedDataAuthority, "Unable to initialize linked data search sub-authority #{subauth}" unless subauth.nil? || subauthority?(subauth)
36
37
  @context = context
38
+ @performance_data = performance_data
37
39
  @language = language_service.preferred_language(user_language: language, authority_language: search_config.language)
38
- url = authority_service.build_url(action_config: search_config, action: :search, action_request: query, substitutions: replacements, subauthority: subauth)
40
+ url = authority_service.build_url(action_config: search_config, action: :search, action_request: query, substitutions: replacements, subauthority: subauth, language: @language)
39
41
  Rails.logger.info "QA Linked Data search url: #{url}"
40
42
  load_graph(url: url)
41
- parse_search_authority_response
43
+ normalize_results
42
44
  end
43
45
 
44
46
  private
45
47
 
46
48
  def load_graph(url:)
49
+ access_start_dt = Time.now.utc
50
+
47
51
  @graph = graph_service.load_graph(url: url)
52
+
53
+ access_end_dt = Time.now.utc
54
+ @access_time_s = access_end_dt - access_start_dt
55
+ Rails.logger.info("Time to receive data from authority: #{access_time_s}s")
56
+ end
57
+
58
+ def normalize_results
59
+ normalize_start_dt = Time.now.utc
60
+
48
61
  @graph = graph_service.filter(graph: @graph, language: language, remove_blanknode_subjects: true)
62
+ results = map_results
63
+ json = convert_results_to_json(results)
64
+
65
+ normalize_end_dt = Time.now.utc
66
+ @normalize_time_s = normalize_end_dt - normalize_start_dt
67
+ Rails.logger.info("Time to convert data to json: #{normalize_time_s}s")
68
+ json = append_performance_data(json) if performance_data?
69
+ json
49
70
  end
50
71
 
51
- def parse_search_authority_response
52
- results = results_mapper_service.map_values(graph: @graph, predicate_map: preds_for_search, sort_key: :sort,
53
- preferred_language: @language, context_map: context_map)
54
- convert_results_to_json(results)
72
+ def map_results
73
+ predicate_map = preds_for_search
74
+ ldpath_map = ldpaths_for_search
75
+
76
+ raise Qa::InvalidConfiguration, "do not specify results using both predicates and ldpath in search configuration for LOD authority #{authority_name} (ldpath is preferred)" if predicate_map.present? && ldpath_map.present? # rubocop:disable Metrics/LineLength
77
+ raise Qa::InvalidConfiguration, "must specify label_ldpath or label_predicate in search configuration for LOD authority #{authority_name} (label_ldpath is preferred)" unless ldpath_map.key?(:label) || predicate_map.key?(:label) # rubocop:disable Metrics/LineLength
78
+
79
+ if predicate_map.present?
80
+ Qa.deprecation_warning(
81
+ in_msg: 'Qa::Authorities::LinkedData::SearchQuery',
82
+ msg: 'defining results using predicates in search config is deprecated; update to define using ldpaths'
83
+ )
84
+ end
85
+
86
+ results_mapper_service.map_values(graph: @graph, prefixes: prefixes, ldpath_map: ldpath_map,
87
+ predicate_map: predicate_map, sort_key: :sort,
88
+ preferred_language: @language, context_map: context_map)
55
89
  end
56
90
 
57
91
  def context_map
@@ -62,11 +96,32 @@ module Qa::Authorities
62
96
  @context == true
63
97
  end
64
98
 
99
+ def performance_data?
100
+ @performance_data == true
101
+ end
102
+
103
+ def ldpaths_for_search
104
+ label_ldpath = search_config.results_label_ldpath
105
+ return {} if label_ldpath.blank?
106
+ ldpaths = { label: label_ldpath, uri: :subject_uri }
107
+ ldpaths[:altlabel] = search_config.results_altlabel_ldpath unless search_config.results_altlabel_ldpath.nil?
108
+ ldpaths[:id] = id_ldpath.present? ? id_ldpath : :subject_uri
109
+ ldpaths[:sort] = sort_ldpath.present? ? sort_ldpath : ldpaths[:label]
110
+ ldpaths
111
+ end
112
+
113
+ def id_ldpath
114
+ @id_ldpath ||= search_config.results_id_ldpath
115
+ end
116
+
117
+ def sort_ldpath
118
+ @sort_ldpath ||= search_config.results_sort_ldpath
119
+ end
120
+
65
121
  def preds_for_search
66
122
  label_pred_uri = search_config.results_label_predicate
67
- raise Qa::InvalidConfiguration, "required label_predicate is missing in search configuration for LOD authority #{auth_name}" if label_pred_uri.nil?
68
- preds = { label: label_pred_uri }
69
- preds[:uri] = :subject_uri
123
+ return {} if label_pred_uri.blank?
124
+ preds = { label: label_pred_uri, uri: :subject_uri }
70
125
  preds[:altlabel] = search_config.results_altlabel_predicate unless search_config.results_altlabel_predicate.nil?
71
126
  preds[:id] = id_predicate.present? ? id_predicate : :subject_uri
72
127
  preds[:sort] = sort_predicate.present? ? sort_predicate : preds[:label]
@@ -111,6 +166,14 @@ module Qa::Authorities
111
166
  lbl = '[' + lbl + ']' if labels.size > 1
112
167
  lbl
113
168
  end
169
+
170
+ def append_performance_data(results)
171
+ performance = { result_count: results.size,
172
+ fetch_time_s: access_time_s,
173
+ normalization_time_s: normalize_time_s,
174
+ total_time_s: (access_time_s + normalize_time_s) }
175
+ { performance: performance, results: results }
176
+ end
114
177
  end
115
178
  end
116
179
  end
@@ -37,5 +37,13 @@ module Qa
37
37
  def default_language
38
38
  @default_language ||= :en
39
39
  end
40
+
41
+ # When true, prevents ldpath requests from making additional network calls. All values will come from the context graph
42
+ # passed to the ldpath request.
43
+ attr_writer :limit_ldpath_to_context
44
+ def limit_ldpath_to_context?
45
+ return true if @limit_ldpath_to_context.nil?
46
+ @limit_ldpath_to_context
47
+ end
40
48
  end
41
49
  end
@@ -1,3 +1,3 @@
1
1
  module Qa
2
- VERSION = "4.0.0".freeze
2
+ VERSION = "4.1.0".freeze
3
3
  end
@@ -81,14 +81,54 @@ describe Qa::LinkedDataTermsController, type: :controller do
81
81
  end
82
82
 
83
83
  describe '#list' do
84
- let(:expected_results) { ['Auth1', 'Auth2', 'Auth3'] }
85
- before do
86
- allow(Qa::LinkedData::AuthorityService).to receive(:authority_names).and_return(expected_results)
84
+ context 'when details=false' do
85
+ let(:expected_results) { ['Auth1', 'Auth2', 'Auth3'] }
86
+ before do
87
+ allow(Qa::LinkedData::AuthorityService).to receive(:authority_names).and_return(expected_results)
88
+ end
89
+ it 'returns list of authorities' do
90
+ get :list
91
+ expect(response).to be_successful
92
+ expect(response.body).to eq expected_results.to_json
93
+ end
87
94
  end
88
- it 'returns list of authorities' do
89
- get :list
90
- expect(response).to be_successful
91
- expect(response.body).to eq expected_results.to_json
95
+
96
+ context 'when details=true' do
97
+ let(:expected_results) do
98
+ [
99
+ {
100
+ "label" => "oclc_fast term (QA)",
101
+ "uri" => "urn:qa:term:oclc_fast",
102
+ "authority" => "oclc_fast",
103
+ "action" => "term",
104
+ "language" => ["en"]
105
+ },
106
+ {
107
+ "label" => "oclc_fast search (QA)",
108
+ "uri" => "urn:qa:search:oclc_fast",
109
+ "authority" => "oclc_fast",
110
+ "action" => "search",
111
+ "language" => ["en"]
112
+ },
113
+ {
114
+ "label" => "oclc_fast search topic (QA)",
115
+ "uri" => "urn:qa:search:oclc_fast:topic",
116
+ "authority" => "oclc_fast",
117
+ "subauthority" => "topic",
118
+ "action" => "search",
119
+ "language" => ["en"]
120
+ }
121
+ ]
122
+ end
123
+
124
+ before do
125
+ allow(Qa::LinkedData::AuthorityService).to receive(:authority_details).and_return(expected_results)
126
+ end
127
+ it 'returns list of authorities' do
128
+ get :list, params: { details: 'true' }
129
+ expect(response).to be_successful
130
+ expect(response.body).to eq expected_results.to_json
131
+ end
92
132
  end
93
133
  end
94
134
 
@@ -254,6 +294,33 @@ describe Qa::LinkedDataTermsController, type: :controller do
254
294
  expect(results.first.key?('context')).to be false
255
295
  end
256
296
  end
297
+
298
+ context 'when requesting performance data' do
299
+ before do
300
+ Qa.config.disable_cors_headers
301
+ stub_request(:get, 'http://experimental.worldcat.org/fast/search?maximumRecords=3&query=cql.any%20all%20%22cornell%22&sortKeys=usage')
302
+ .to_return(status: 200, body: webmock_fixture('lod_oclc_all_query_3_results.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
303
+ end
304
+ it "returns basic data + performance data when performance_data='true'" do
305
+ get :search, params: { q: 'cornell', vocab: 'OCLC_FAST', maximumRecords: '3', performance_data: 'true' }
306
+ expect(response).to be_successful
307
+ results = JSON.parse(response.body)
308
+ expect(results).to be_kind_of Hash
309
+ expect(results.keys).to match_array ['performance', 'results']
310
+ expect(results['performance'].keys).to match_array ['result_count', 'fetch_time_s', 'normalization_time_s', 'total_time_s']
311
+ expect(results['performance']['total_time_s']).to eq results['performance']['fetch_time_s'] + results['performance']['normalization_time_s']
312
+ expect(results['performance']['result_count']).to eq 3
313
+ expect(results['results'].count).to eq 3
314
+ end
315
+
316
+ it "returns basic data only when performance_data='false'" do
317
+ get :search, params: { q: 'cornell', vocab: 'OCLC_FAST', maximumRecords: '3', performance_data: 'false' }
318
+ expect(response).to be_successful
319
+ results = JSON.parse(response.body)
320
+ expect(results).to be_kind_of Array
321
+ expect(results.size).to eq 3
322
+ end
323
+ end
257
324
  end
258
325
 
259
326
  describe '#show' do
@@ -378,48 +445,75 @@ describe Qa::LinkedDataTermsController, type: :controller do
378
445
  .to_return(status: 200, body: webmock_fixture('lod_loc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
379
446
  end
380
447
  it 'succeeds and defaults to json content type' do
381
- get :show, params: { id: 'sh85118553', vocab: 'LOC', subauthority: 'subjects' }
448
+ get :show, params: { id: 'sh 85118553', vocab: 'LOC', subauthority: 'subjects' }
382
449
  expect(response).to be_successful
383
450
  expect(response.content_type).to eq 'application/json'
384
451
  end
385
452
  end
386
453
  end
454
+
455
+ context 'when requesting performance data' do
456
+ before do
457
+ stub_request(:get, 'http://id.loc.gov/authorities/subjects/sh85118553')
458
+ .to_return(status: 200, body: webmock_fixture('lod_loc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
459
+ end
460
+ it "returns basic data + performance data when performance_data='true'" do
461
+ get :show, params: { id: 'sh 85118553', vocab: 'LOC', subauthority: 'subjects', performance_data: 'true' }
462
+ expect(response).to be_successful
463
+ results = JSON.parse(response.body)
464
+ expect(results).to be_kind_of Hash
465
+ expect(results.keys).to match_array ['performance', 'results']
466
+ expect(results['performance'].keys).to match_array ['predicate_count', 'fetch_time_s', 'normalization_time_s', 'total_time_s']
467
+ expect(results['performance']['total_time_s']).to eq results['performance']['fetch_time_s'] + results['performance']['normalization_time_s']
468
+ expect(results['performance']['predicate_count']).to eq 15
469
+ expect(results['results']['predicates'].count).to eq 15
470
+ end
471
+
472
+ it "returns basic data only when performance_data='false'" do
473
+ get :show, params: { id: 'sh 85118553', vocab: 'LOC', subauthority: 'subjects', performance_data: 'false' }
474
+ expect(response).to be_successful
475
+ results = JSON.parse(response.body)
476
+ expect(results).to be_kind_of Hash
477
+ expect(results.keys).not_to include('performance')
478
+ expect(results['predicates'].size).to eq 15
479
+ end
480
+ end
387
481
  end
388
482
 
389
483
  describe '#fetch' do
390
484
  context 'producing internal server error' do
391
485
  context 'when server returns 500' do
392
486
  before do
393
- stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369').to_return(status: 500)
487
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369').to_return(status: 500)
394
488
  end
395
489
  it 'returns 500' do
396
- expect(Rails.logger).to receive(:warn).with("Internal Server Error - Fetch term http://test.org/530369 unsuccessful for authority LOD_TERM_URI_PARAM_CONFIG")
397
- get :fetch, params: { vocab: 'LOD_TERM_URI_PARAM_CONFIG', uri: 'http://test.org/530369' }
490
+ expect(Rails.logger).to receive(:warn).with("Internal Server Error - Fetch term http://id.worldcat.org/fast/530369 unsuccessful for authority LOD_TERM_URI_PARAM_CONFIG")
491
+ get :fetch, params: { vocab: 'LOD_TERM_URI_PARAM_CONFIG', uri: 'http://id.worldcat.org/fast/530369' }
398
492
  expect(response.code).to eq('500')
399
493
  end
400
494
  end
401
495
 
402
496
  context 'when rdf format error' do
403
497
  before do
404
- stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369').to_return(status: 200)
498
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369').to_return(status: 200)
405
499
  allow(RDF::Graph).to receive(:load).and_raise(RDF::FormatError)
406
500
  end
407
501
  it 'returns 500' do
408
- msg = "RDF Format Error - Results from fetch term http://test.org/530369 for authority LOD_TERM_URI_PARAM_CONFIG was not identified as a valid RDF format. " \
502
+ msg = "RDF Format Error - Results from fetch term http://id.worldcat.org/fast/530369 for authority LOD_TERM_URI_PARAM_CONFIG was not identified as a valid RDF format. " \
409
503
  "You may need to include the linkeddata gem."
410
504
  expect(Rails.logger).to receive(:warn).with(msg)
411
- get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
505
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
412
506
  expect(response.code).to eq('500')
413
507
  end
414
508
  end
415
509
 
416
510
  context "when error isn't specifically handled" do
417
511
  before do
418
- stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369').to_return(status: 501)
512
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369').to_return(status: 501)
419
513
  end
420
514
  it 'returns 500' do
421
- expect(Rails.logger).to receive(:warn).with("Internal Server Error - Fetch term http://test.org/530369 unsuccessful for authority LOD_TERM_URI_PARAM_CONFIG")
422
- get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
515
+ expect(Rails.logger).to receive(:warn).with("Internal Server Error - Fetch term http://id.worldcat.org/fast/530369 unsuccessful for authority LOD_TERM_URI_PARAM_CONFIG")
516
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
423
517
  expect(response.code).to eq('500')
424
518
  end
425
519
  end
@@ -427,11 +521,11 @@ describe Qa::LinkedDataTermsController, type: :controller do
427
521
 
428
522
  context 'when service unavailable' do
429
523
  before do
430
- stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369').to_return(status: 503)
524
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369').to_return(status: 503)
431
525
  end
432
526
  it 'returns 503' do
433
- expect(Rails.logger).to receive(:warn).with("Service Unavailable - Fetch term http://test.org/530369 unsuccessful for authority LOD_TERM_URI_PARAM_CONFIG")
434
- get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
527
+ expect(Rails.logger).to receive(:warn).with("Service Unavailable - Fetch term http://id.worldcat.org/fast/530369 unsuccessful for authority LOD_TERM_URI_PARAM_CONFIG")
528
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
435
529
  expect(response.code).to eq('503')
436
530
  end
437
531
  end
@@ -450,19 +544,19 @@ describe Qa::LinkedDataTermsController, type: :controller do
450
544
  context 'in LOD_TERM_URI_PARAM_CONFIG authority' do
451
545
  context 'term found' do
452
546
  before do
453
- stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369')
547
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369')
454
548
  .to_return(status: 200, body: webmock_fixture('lod_oclc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
455
549
  end
456
550
 
457
551
  it 'succeeds and defaults to json content type' do
458
- get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
552
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
459
553
  expect(response).to be_successful
460
554
  expect(response.content_type).to eq 'application/json'
461
555
  end
462
556
 
463
557
  context 'and it was requested as json' do
464
558
  it 'succeeds and returns term data as json content type' do
465
- get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG', format: 'json' }
559
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG', format: 'json' }
466
560
  expect(response).to be_successful
467
561
  expect(response.content_type).to eq 'application/json'
468
562
  end
@@ -470,7 +564,7 @@ describe Qa::LinkedDataTermsController, type: :controller do
470
564
 
471
565
  context 'and it was requested as jsonld' do
472
566
  it 'succeeds and returns term data as jsonld content type' do
473
- get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG', format: 'jsonld' }
567
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG', format: 'jsonld' }
474
568
  expect(response).to be_successful
475
569
  expect(response.content_type).to eq 'application/ld+json'
476
570
  expect(JSON.parse(response.body).keys).to match_array ["@context", "@graph"]
@@ -479,11 +573,11 @@ describe Qa::LinkedDataTermsController, type: :controller do
479
573
 
480
574
  context 'blank nodes not included in predicates list' do
481
575
  before do
482
- stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369wbn')
576
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369wbn')
483
577
  .to_return(status: 200, body: webmock_fixture('lod_term_with_blanknode_objects.nt'), headers: { 'Content-Type' => 'application/n-triples' })
484
578
  end
485
579
  it 'succeeds' do
486
- get :fetch, params: { uri: 'http://test.org/530369wbn', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
580
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369wbn', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
487
581
  expect(response).to be_successful
488
582
  end
489
583
  end
@@ -492,11 +586,11 @@ describe Qa::LinkedDataTermsController, type: :controller do
492
586
  context 'when cors headers are enabled' do
493
587
  before do
494
588
  Qa.config.enable_cors_headers
495
- stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369')
589
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369')
496
590
  .to_return(status: 200, body: webmock_fixture('lod_oclc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
497
591
  end
498
592
  it 'Access-Control-Allow-Origin is *' do
499
- get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
593
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
500
594
  expect(response.headers['Access-Control-Allow-Origin']).to eq '*'
501
595
  end
502
596
  end
@@ -504,15 +598,42 @@ describe Qa::LinkedDataTermsController, type: :controller do
504
598
  context 'when cors headers are disabled' do
505
599
  before do
506
600
  Qa.config.disable_cors_headers
507
- stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369')
601
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369')
508
602
  .to_return(status: 200, body: webmock_fixture('lod_oclc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
509
603
  end
510
604
  it 'Access-Control-Allow-Origin is not present' do
511
- get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
605
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
512
606
  expect(response.headers.key?('Access-Control-Allow-Origin')).to be false
513
607
  end
514
608
  end
515
609
  end
610
+
611
+ context 'when requesting performance data' do
612
+ before do
613
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369')
614
+ .to_return(status: 200, body: webmock_fixture('lod_oclc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
615
+ end
616
+ it "returns basic data + performance data when performance_data='true'" do
617
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG', performance_data: 'true' }
618
+ expect(response).to be_successful
619
+ results = JSON.parse(response.body)
620
+ expect(results).to be_kind_of Hash
621
+ expect(results.keys).to match_array ['performance', 'results']
622
+ expect(results['performance'].keys).to match_array ['predicate_count', 'fetch_time_s', 'normalization_time_s', 'total_time_s']
623
+ expect(results['performance']['total_time_s']).to eq results['performance']['fetch_time_s'] + results['performance']['normalization_time_s']
624
+ expect(results['performance']['predicate_count']).to eq 7
625
+ expect(results['results']['predicates'].count).to eq 7
626
+ end
627
+
628
+ it "returns basic data only when performance_data='false'" do
629
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG', performance_data: 'false' }
630
+ expect(response).to be_successful
631
+ results = JSON.parse(response.body)
632
+ expect(results).to be_kind_of Hash
633
+ expect(results.keys).not_to include('performance')
634
+ expect(results['predicates'].size).to eq 7
635
+ end
636
+ end
516
637
  end
517
638
 
518
639
  describe '#reload' do