qa 4.0.0 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/app/controllers/qa/linked_data_terms_controller.rb +30 -9
  4. data/app/controllers/qa/terms_controller.rb +3 -2
  5. data/app/models/qa/linked_data/config/context_property_map.rb +6 -25
  6. data/app/services/qa/iri_template_service.rb +32 -24
  7. data/app/services/qa/linked_data/authority_service.rb +8 -0
  8. data/app/services/qa/linked_data/authority_url_service.rb +27 -8
  9. data/app/services/qa/linked_data/deep_sort_service.rb +3 -2
  10. data/app/services/qa/linked_data/graph_service.rb +13 -0
  11. data/app/services/qa/linked_data/language_service.rb +12 -0
  12. data/app/services/qa/linked_data/language_sort_service.rb +7 -2
  13. data/app/services/qa/linked_data/ldpath_service.rb +40 -0
  14. data/app/services/qa/linked_data/mapper/graph_ldpath_mapper_service.rb +49 -0
  15. data/app/services/qa/linked_data/mapper/graph_mapper_service.rb +3 -11
  16. data/app/services/qa/linked_data/mapper/graph_predicate_mapper_service.rb +40 -0
  17. data/app/services/qa/linked_data/mapper/search_results_mapper_service.rb +58 -11
  18. data/app/services/qa/linked_data/mapper/term_results_mapper_service.rb +80 -0
  19. data/config/authorities/linked_data/loc.json +13 -7
  20. data/config/authorities/linked_data/oclc_fast.json +13 -8
  21. data/lib/generators/qa/discogs/USAGE +10 -0
  22. data/lib/generators/qa/discogs/discogs_generator.rb +12 -0
  23. data/lib/generators/qa/discogs/templates/config/discogs-formats.yml +346 -0
  24. data/lib/generators/qa/discogs/templates/config/discogs-genres.yml +627 -0
  25. data/lib/generators/qa/install/templates/config/initializers/qa.rb +4 -0
  26. data/lib/qa.rb +6 -0
  27. data/lib/qa/authorities.rb +2 -0
  28. data/lib/qa/authorities/discogs.rb +28 -0
  29. data/lib/qa/authorities/discogs/discogs_instance_builder.rb +145 -0
  30. data/lib/qa/authorities/discogs/discogs_translation.rb +126 -0
  31. data/lib/qa/authorities/discogs/discogs_utils.rb +89 -0
  32. data/lib/qa/authorities/discogs/discogs_works_builder.rb +153 -0
  33. data/lib/qa/authorities/discogs/generic_authority.rb +151 -0
  34. data/lib/qa/authorities/discogs_subauthority.rb +9 -0
  35. data/lib/qa/authorities/linked_data/config.rb +7 -3
  36. data/lib/qa/authorities/linked_data/config/search_config.rb +99 -11
  37. data/lib/qa/authorities/linked_data/config/term_config.rb +112 -8
  38. data/lib/qa/authorities/linked_data/find_term.rb +154 -84
  39. data/lib/qa/authorities/linked_data/search_query.rb +76 -13
  40. data/lib/qa/configuration.rb +8 -0
  41. data/lib/qa/version.rb +1 -1
  42. data/spec/controllers/linked_data_terms_controller_spec.rb +151 -30
  43. data/spec/controllers/terms_controller_spec.rb +4 -0
  44. data/spec/features/linked_data/language_spec.rb +298 -0
  45. data/spec/fixtures/authorities/linked_data/lod_full_config.json +21 -5
  46. data/spec/fixtures/authorities/linked_data/lod_lang_defaults.json +4 -4
  47. data/spec/fixtures/authorities/linked_data/lod_lang_multi_defaults.json +4 -4
  48. data/spec/fixtures/authorities/linked_data/lod_lang_no_defaults.json +4 -5
  49. data/spec/fixtures/authorities/linked_data/lod_lang_param.json +4 -4
  50. data/spec/fixtures/authorities/linked_data/lod_term_uri_param_config.json +1 -1
  51. data/spec/fixtures/discogs-find-response-json.json +1 -0
  52. data/spec/fixtures/discogs-find-response-jsonld-master.json +1 -0
  53. data/spec/fixtures/discogs-find-response-jsonld-release.json +1 -0
  54. data/spec/fixtures/discogs-id-matches-master.json +1 -0
  55. data/spec/fixtures/discogs-id-matches-release.json +1 -0
  56. data/spec/fixtures/discogs-id-not-found-master.json +1 -0
  57. data/spec/fixtures/discogs-id-not-found-release.json +1 -0
  58. data/spec/fixtures/discogs-search-response-no-auth.json +1 -0
  59. data/spec/fixtures/discogs-search-response-no-subauth.json +1 -0
  60. data/spec/fixtures/discogs-search-response-subauth.json +1 -0
  61. data/spec/fixtures/lod_lang_search_enesfrde.rdf.xml +60 -0
  62. data/spec/fixtures/lod_lang_search_sv.rdf.xml +42 -0
  63. data/spec/fixtures/lod_loc_term_found.rdf.xml +5 -0
  64. data/spec/lib/authorities/discogs/generic_authority_spec.rb +235 -0
  65. data/spec/lib/authorities/discogs_spec.rb +17 -0
  66. data/spec/lib/authorities/linked_data/config_spec.rb +68 -5
  67. data/spec/lib/authorities/linked_data/find_term_spec.rb +298 -3
  68. data/spec/lib/authorities/linked_data/generic_authority_spec.rb +46 -485
  69. data/spec/lib/authorities/linked_data/search_config_spec.rb +154 -3
  70. data/spec/lib/authorities/linked_data/search_query_spec.rb +240 -3
  71. data/spec/lib/authorities/linked_data/term_config_spec.rb +193 -5
  72. data/spec/lib/configuration_spec.rb +18 -0
  73. data/spec/models/linked_data/config/context_property_map_spec.rb +3 -31
  74. data/spec/services/iri_template_service_spec.rb +54 -12
  75. data/spec/{lib/authorities → services}/linked_data/authority_service_spec.rb +47 -0
  76. data/spec/services/linked_data/language_service_spec.rb +52 -11
  77. data/spec/services/linked_data/ldpath_service_spec.rb +61 -0
  78. data/spec/services/linked_data/mapper/graph_ldpath_mapper_service_spec.rb +118 -0
  79. data/spec/services/linked_data/mapper/graph_predicate_mapper_service_spec.rb +110 -0
  80. data/spec/services/linked_data/mapper/term_results_mapper_service_spec.rb +94 -0
  81. data/spec/spec_helper.rb +1 -1
  82. data/spec/support/matchers/include_hash.rb +5 -0
  83. data/spec/test_app_templates/lib/generators/test_app_generator.rb +4 -0
  84. metadata +73 -5
  85. data/lib/qa/authorities/linked_data/rdf_helper.rb +0 -49
@@ -3,24 +3,29 @@
3
3
  module Qa::Authorities
4
4
  module LinkedData
5
5
  class FindTerm
6
- include Qa::Authorities::LinkedData::RdfHelper
6
+ class_attribute :authority_service, :graph_service, :language_service, :language_sort_service, :results_mapper_service
7
+ self.authority_service = Qa::LinkedData::AuthorityUrlService
8
+ self.graph_service = Qa::LinkedData::GraphService
9
+ self.language_service = Qa::LinkedData::LanguageService
10
+ self.language_sort_service = Qa::LinkedData::LanguageSortService
11
+ self.results_mapper_service = Qa::LinkedData::Mapper::TermResultsMapperService
7
12
 
8
13
  # @param [TermConfig] term_config The term portion of the config
9
14
  def initialize(term_config)
10
15
  @term_config = term_config
11
16
  end
12
17
 
13
- attr_reader :term_config, :full_graph, :filtered_graph, :language
14
- private :full_graph, :filtered_graph, :language
18
+ attr_reader :term_config, :full_graph, :filtered_graph, :language, :id, :access_time_s, :normalize_time_s
19
+ private :full_graph, :filtered_graph, :language, :id, :access_time_s, :normalize_time_s
15
20
 
16
- delegate :term_subauthority?, to: :term_config
21
+ delegate :term_subauthority?, :prefixes, :authority_name, to: :term_config
17
22
 
18
23
  # Find a single term in a linked data authority
19
24
  # @param [String] the id of the term to fetch
20
25
  # @param [Symbol] (optional) language: language used to select literals when multi-language is supported (e.g. :en, :fr, etc.)
21
26
  # @param [Hash] (optional) replacements: replacement values with { pattern_name (defined in YAML config) => value }
22
27
  # @param [String] subauth: the subauthority from which to fetch the term
23
- # @return [String] json results
28
+ # @return [Hash] json results
24
29
  # @example Json Results for Linked Data Term
25
30
  # { "uri":"http://id.worldcat.org/fast/530369",
26
31
  # "id":"530369","label":"Cornell University",
@@ -34,102 +39,159 @@ module Qa::Authorities
34
39
  # "http://schema.org/name":["Cornell University","Ithaca (N.Y.). Cornell University"],
35
40
  # "http://www.w3.org/2004/02/skos/core#altLabel":["Ithaca (N.Y.). Cornell University"],
36
41
  # "http://schema.org/sameAs":["http://id.loc.gov/authorities/names/n79021621","https://viaf.org/viaf/126293486"] } }
37
- def find(id, language: nil, replacements: {}, subauth: nil, jsonld: false)
42
+ def find(id, language: nil, replacements: {}, subauth: nil, jsonld: false, performance_data: false) # rubocop:disable Metrics/ParameterLists
38
43
  raise Qa::InvalidLinkedDataAuthority, "Unable to initialize linked data term sub-authority #{subauth}" unless subauth.nil? || term_subauthority?(subauth)
39
- @language = Qa::LinkedData::LanguageService.preferred_language(user_language: language, authority_language: term_config.term_language)
40
- url = Qa::LinkedData::AuthorityUrlService.build_url(action_config: term_config, action: :term, action_request: id, substitutions: replacements, subauthority: subauth)
44
+ @language = language_service.preferred_language(user_language: language, authority_language: term_config.term_language)
45
+ @id = id
46
+ @performance_data = performance_data
47
+ @jsonld = jsonld
48
+ url = authority_service.build_url(action_config: term_config, action: :term, action_request: normalize_id, substitutions: replacements, subauthority: subauth, language: @language)
41
49
  Rails.logger.info "QA Linked Data term url: #{url}"
42
50
  load_graph(url: url)
43
- return "{}" unless full_graph.size.positive?
44
- return full_graph.dump(:jsonld, standard_prefixes: true) if jsonld
45
- parse_term_authority_response(id)
51
+ normalize_results
46
52
  end
47
53
 
48
54
  private
49
55
 
50
56
  def load_graph(url:)
51
- # @graph = Qa::LinkedData::GraphService.load_graph(url: url)
52
- @full_graph = Qa::LinkedData::GraphService.load_graph(url: url)
53
- return unless @full_graph.size.positive?
54
- @filtered_graph = Qa::LinkedData::GraphService.deep_copy(graph: @full_graph)
55
- @filtered_graph = Qa::LinkedData::GraphService.filter(graph: @filtered_graph, language: language) unless language.blank?
57
+ access_start_dt = Time.now.utc
58
+
59
+ @full_graph = graph_service.load_graph(url: url)
60
+
61
+ access_end_dt = Time.now.utc
62
+ @access_time_s = access_end_dt - access_start_dt
63
+ Rails.logger.info("Time to receive data from authority: #{access_time_s}s")
56
64
  end
57
65
 
58
- def parse_term_authority_response(id)
59
- results = extract_preds(filtered_graph, preds_for_term)
60
- consolidated_results = consolidate_term_results(results)
61
- json_results = convert_term_to_json(consolidated_results)
62
- termhash = select_json_result_for_id(json_results, id)
63
- predicates_hash = predicates_with_subject_uri(termhash[:uri])
64
- termhash['predicates'] = predicates_hash unless predicates_hash.length <= 0
65
- termhash
66
+ def normalize_results
67
+ normalize_start_dt = Time.now.utc
68
+ normalize_end_dt = Time.now.utc
69
+
70
+ json = perform_normalization
71
+
72
+ @normalize_time_s = normalize_end_dt - normalize_start_dt
73
+ Rails.logger.info("Time to convert data to json: #{normalize_time_s}s")
74
+ json = append_performance_data(json) if performance_data? && !jsonld?
75
+ json
66
76
  end
67
77
 
68
- def preds_for_term
69
- { required: required_term_preds, optional: optional_term_preds }
78
+ def perform_normalization
79
+ return "{}" unless full_graph.size.positive?
80
+ return full_graph.dump(:jsonld, standard_prefixes: true) if jsonld?
81
+
82
+ filter_graph
83
+ results = map_results
84
+ convert_results_to_json(results)
70
85
  end
71
86
 
72
- def required_term_preds
73
- label_pred_uri = term_config.term_results_label_predicate
74
- raise Qa::InvalidConfiguration, "required label_predicate is missing in configuration for LOD authority #{auth_name}" if label_pred_uri.nil?
75
- { label: label_pred_uri }
76
- end
77
-
78
- def optional_term_preds
79
- preds = {}
80
- preds[:altlabel] = term_config.term_results_altlabel_predicate unless term_config.term_results_altlabel_predicate.nil?
81
- preds[:id] = term_config.term_results_id_predicate unless term_config.term_results_id_predicate.nil?
82
- preds[:narrower] = term_config.term_results_narrower_predicate unless term_config.term_results_narrower_predicate.nil?
83
- preds[:broader] = term_config.term_results_broader_predicate unless term_config.term_results_broader_predicate.nil?
84
- preds[:sameas] = term_config.term_results_sameas_predicate unless term_config.term_results_sameas_predicate.nil?
85
- preds
86
- end
87
-
88
- def consolidate_term_results(results) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize # TODO: Explore a way to simplify
89
- consolidated_results = {}
90
- results.each do |statement|
91
- stmt_hash = statement.to_h
92
- uri = stmt_hash[:uri].to_s
93
- consolidated_hash = init_consolidated_hash(consolidated_results, uri, stmt_hash[:id].to_s)
94
-
95
- consolidated_hash[:label] = object_value(stmt_hash, consolidated_hash, :label, false)
96
- altlabel = object_value(stmt_hash, consolidated_hash, :altlabel, false)
97
- narrower = object_value(stmt_hash, consolidated_hash, :narrower)
98
- broader = object_value(stmt_hash, consolidated_hash, :broader)
99
- sameas = object_value(stmt_hash, consolidated_hash, :sameas)
100
-
101
- consolidated_hash[:altlabel] = altlabel unless altlabel.nil?
102
- consolidated_hash[:narrower] = narrower unless narrower.nil?
103
- consolidated_hash[:broader] = broader unless broader.nil?
104
- consolidated_hash[:sameas] = sameas unless sameas.nil?
105
- consolidated_results[uri] = consolidated_hash
106
- end
107
- consolidated_results.each do |res|
108
- consolidated_hash = res[1]
109
- consolidated_hash[:label] = sort_string_by_language consolidated_hash[:label]
110
- consolidated_hash[:altlabel] = sort_string_by_language consolidated_hash[:altlabel]
111
- consolidated_hash[:sort] = sort_string_by_language consolidated_hash[:sort]
112
- end
113
- consolidated_results
114
- end
115
-
116
- def convert_term_to_json(consolidated_results)
117
- json_results = []
118
- consolidated_results.each do |uri, h|
119
- json_hash = { uri: uri, id: h[:id], label: h[:label] }
120
- json_hash[:altlabel] = h[:altlabel] unless h[:altlabel].nil?
121
- json_hash[:narrower] = h[:narrower] unless h[:narrower].nil?
122
- json_hash[:broader] = h[:broader] unless h[:broader].nil?
123
- json_hash[:sameas] = h[:sameas] unless h[:sameas].nil?
124
- json_results << json_hash
87
+ def filter_graph
88
+ @filtered_graph = graph_service.deep_copy(graph: @full_graph)
89
+ @filtered_graph = graph_service.filter(graph: @filtered_graph, language: language) unless language.blank?
90
+ end
91
+
92
+ def map_results
93
+ predicate_map = preds_for_term
94
+ ldpath_map = ldpaths_for_term
95
+
96
+ raise Qa::InvalidConfiguration, "do not specify results using both predicates and ldpath in term configuration for LOD authority #{authority_name} (ldpath is preferred)" if predicate_map.present? && ldpath_map.present? # rubocop:disable Metrics/LineLength
97
+ raise Qa::InvalidConfiguration, "must specify label_ldpath or label_predicate in term configuration for LOD authority #{authority_name} (label_ldpath is preferred)" unless ldpath_map.key?(:label) || predicate_map.key?(:label) # rubocop:disable Metrics/LineLength
98
+
99
+ if predicate_map.present?
100
+ Qa.deprecation_warning(
101
+ in_msg: 'Qa::Authorities::LinkedData::FindTerm',
102
+ msg: 'defining results using predicates in term config is deprecated; update to define using ldpaths'
103
+ )
125
104
  end
126
- json_results
105
+
106
+ results_mapper_service.map_values(graph: @filtered_graph, subject_uri: uri, prefixes: prefixes,
107
+ ldpath_map: ldpaths_for_term, predicate_map: preds_for_term)
108
+ end
109
+
110
+ def normalize_id
111
+ return id if expects_uri?
112
+ authority_name.to_s.casecmp('loc').zero? ? id.delete(' ') : id
113
+ end
114
+
115
+ def expects_uri?
116
+ term_config.term_id_expects_uri?
117
+ end
118
+
119
+ def uri
120
+ return @uri if @uri.present?
121
+ return @uri = RDF::URI.new(id) if expects_uri?
122
+ @uri = graph_service.subjects_for_object_value(graph: @filtered_graph, predicate: RDF::URI.new(term_config.term_results_id_predicate), object_value: id.gsub('%20', ' ')).first
123
+ end
124
+
125
+ def ldpaths_for_term
126
+ label_ldpath = term_config.term_results_label_ldpath
127
+ return {} if label_ldpath.blank?
128
+ ldpaths = { label: label_ldpath }
129
+ ldpaths.merge(optional_ldpaths)
130
+ end
131
+
132
+ def optional_ldpaths
133
+ opt_ldpaths = {}
134
+ opt_ldpaths[:altlabel] = term_config.term_results_altlabel_ldpath
135
+ opt_ldpaths[:id] = term_config.term_results_id_ldpath
136
+ opt_ldpaths[:narrower] = term_config.term_results_narrower_ldpath
137
+ opt_ldpaths[:broader] = term_config.term_results_broader_ldpath
138
+ opt_ldpaths[:sameas] = term_config.term_results_sameas_ldpath
139
+ opt_ldpaths.delete_if { |_k, v| v.blank? }
140
+ end
141
+
142
+ def jsonld?
143
+ @jsonld == true
144
+ end
145
+
146
+ def performance_data?
147
+ @performance_data == true
148
+ end
149
+
150
+ def preds_for_term
151
+ label_pred_uri = term_config.term_results_label_predicate
152
+ return {} if label_pred_uri.blank?
153
+ preds = { label: label_pred_uri }
154
+ preds.merge(optional_preds)
127
155
  end
128
156
 
129
- def select_json_result_for_id(json_results, id)
130
- json_results.select! { |r| r[:uri].include? id } if json_results.size > 1
131
- json_results.select! { |r| r[:uri].ends_with? id } if json_results.size > 1
132
- json_results.first
157
+ def optional_preds
158
+ opt_preds = {}
159
+ opt_preds[:altlabel] = term_config.term_results_altlabel_predicate
160
+ opt_preds[:id] = term_config.term_results_id_predicate
161
+ opt_preds[:narrower] = term_config.term_results_narrower_predicate
162
+ opt_preds[:broader] = term_config.term_results_broader_predicate
163
+ opt_preds[:sameas] = term_config.term_results_sameas_predicate
164
+ opt_preds.delete_if { |_k, v| v.blank? }
165
+ end
166
+
167
+ def convert_results_to_json(results)
168
+ json_hash = { uri: uri.to_s }
169
+ json_hash[:id] = results.key?(:id) && results[:id].present? ? results[:id].first.to_s : uri.to_s
170
+ json_hash[:label] = sort_literals(results, :label)
171
+ json_hash.merge!(optional_results_to_json(results))
172
+ predicates_hash = predicates_with_subject_uri(uri)
173
+ json_hash['predicates'] = predicates_hash if predicates_hash.present?
174
+ json_hash
175
+ end
176
+
177
+ def optional_results_to_json(results)
178
+ opt_results_json = {}
179
+ opt_results_json[:altlabel] = sort_literals(results, :altlabel)
180
+ opt_results_json[:narrower] = extract_result(results, :narrower)
181
+ opt_results_json[:broader] = extract_result(results, :broader)
182
+ opt_results_json[:sameas] = extract_result(results, :sameas)
183
+ opt_results_json.delete_if { |_k, v| v.blank? }
184
+ end
185
+
186
+ def extract_result(results, key)
187
+ return nil unless results.key?(key) && results[key].present?
188
+ results[key].map(&:to_s)
189
+ end
190
+
191
+ def sort_literals(results, key)
192
+ return nil unless results.key? key
193
+ return [] if results[key].blank?
194
+ language_sort_service.new(results[key], language).uniq_sorted_strings
133
195
  end
134
196
 
135
197
  def predicates_with_subject_uri(expected_uri) # rubocop:disable Metrics/MethodLength
@@ -152,6 +214,14 @@ module Qa::Authorities
152
214
  end
153
215
  predicates_hash
154
216
  end
217
+
218
+ def append_performance_data(results)
219
+ performance = { predicate_count: results['predicates'].size,
220
+ fetch_time_s: access_time_s,
221
+ normalization_time_s: normalize_time_s,
222
+ total_time_s: (access_time_s + normalize_time_s) }
223
+ { performance: performance, results: results }
224
+ end
155
225
  end
156
226
  end
157
227
  end
@@ -15,10 +15,10 @@ module Qa::Authorities
15
15
  @search_config = search_config
16
16
  end
17
17
 
18
- attr_reader :search_config, :graph, :language
19
- private :graph, :language
18
+ attr_reader :search_config, :graph, :language, :access_time_s, :normalize_time_s
19
+ private :graph, :language, :access_time_s, :normalize_time_s
20
20
 
21
- delegate :subauthority?, :supports_sort?, to: :search_config
21
+ delegate :subauthority?, :supports_sort?, :prefixes, :authority_name, to: :search_config
22
22
 
23
23
  # Search a linked data authority
24
24
  # @praram [String] the query
@@ -26,32 +26,66 @@ module Qa::Authorities
26
26
  # @param replacements [Hash] (optional) replacement values with { pattern_name (defined in YAML config) => value }
27
27
  # @param subauth [String] (optional) the subauthority to query
28
28
  # @param context [Boolean] (optional) true if context should be returned with the results; otherwise, false (default: false)
29
+ # @param performance_data [Boolean] (optional) true if include_performance_data should be returned with the results; otherwise, false (default: false)
29
30
  # @return [String] json results
30
31
  # @example Json Results for Linked Data Search
31
32
  # [ {"uri":"http://id.worldcat.org/fast/5140","id":"5140","label":"Cornell, Joseph"},
32
33
  # {"uri":"http://id.worldcat.org/fast/72456","id":"72456","label":"Cornell, Sarah Maria, 1802-1832"},
33
34
  # {"uri":"http://id.worldcat.org/fast/409667","id":"409667","label":"Cornell, Ezra, 1807-1874"} ]
34
- def search(query, language: nil, replacements: {}, subauth: nil, context: false)
35
+ def search(query, language: nil, replacements: {}, subauth: nil, context: false, performance_data: false) # rubocop:disable Metrics/ParameterLists
35
36
  raise Qa::InvalidLinkedDataAuthority, "Unable to initialize linked data search sub-authority #{subauth}" unless subauth.nil? || subauthority?(subauth)
36
37
  @context = context
38
+ @performance_data = performance_data
37
39
  @language = language_service.preferred_language(user_language: language, authority_language: search_config.language)
38
- url = authority_service.build_url(action_config: search_config, action: :search, action_request: query, substitutions: replacements, subauthority: subauth)
40
+ url = authority_service.build_url(action_config: search_config, action: :search, action_request: query, substitutions: replacements, subauthority: subauth, language: @language)
39
41
  Rails.logger.info "QA Linked Data search url: #{url}"
40
42
  load_graph(url: url)
41
- parse_search_authority_response
43
+ normalize_results
42
44
  end
43
45
 
44
46
  private
45
47
 
46
48
  def load_graph(url:)
49
+ access_start_dt = Time.now.utc
50
+
47
51
  @graph = graph_service.load_graph(url: url)
52
+
53
+ access_end_dt = Time.now.utc
54
+ @access_time_s = access_end_dt - access_start_dt
55
+ Rails.logger.info("Time to receive data from authority: #{access_time_s}s")
56
+ end
57
+
58
+ def normalize_results
59
+ normalize_start_dt = Time.now.utc
60
+
48
61
  @graph = graph_service.filter(graph: @graph, language: language, remove_blanknode_subjects: true)
62
+ results = map_results
63
+ json = convert_results_to_json(results)
64
+
65
+ normalize_end_dt = Time.now.utc
66
+ @normalize_time_s = normalize_end_dt - normalize_start_dt
67
+ Rails.logger.info("Time to convert data to json: #{normalize_time_s}s")
68
+ json = append_performance_data(json) if performance_data?
69
+ json
49
70
  end
50
71
 
51
- def parse_search_authority_response
52
- results = results_mapper_service.map_values(graph: @graph, predicate_map: preds_for_search, sort_key: :sort,
53
- preferred_language: @language, context_map: context_map)
54
- convert_results_to_json(results)
72
+ def map_results
73
+ predicate_map = preds_for_search
74
+ ldpath_map = ldpaths_for_search
75
+
76
+ raise Qa::InvalidConfiguration, "do not specify results using both predicates and ldpath in search configuration for LOD authority #{authority_name} (ldpath is preferred)" if predicate_map.present? && ldpath_map.present? # rubocop:disable Metrics/LineLength
77
+ raise Qa::InvalidConfiguration, "must specify label_ldpath or label_predicate in search configuration for LOD authority #{authority_name} (label_ldpath is preferred)" unless ldpath_map.key?(:label) || predicate_map.key?(:label) # rubocop:disable Metrics/LineLength
78
+
79
+ if predicate_map.present?
80
+ Qa.deprecation_warning(
81
+ in_msg: 'Qa::Authorities::LinkedData::SearchQuery',
82
+ msg: 'defining results using predicates in search config is deprecated; update to define using ldpaths'
83
+ )
84
+ end
85
+
86
+ results_mapper_service.map_values(graph: @graph, prefixes: prefixes, ldpath_map: ldpath_map,
87
+ predicate_map: predicate_map, sort_key: :sort,
88
+ preferred_language: @language, context_map: context_map)
55
89
  end
56
90
 
57
91
  def context_map
@@ -62,11 +96,32 @@ module Qa::Authorities
62
96
  @context == true
63
97
  end
64
98
 
99
+ def performance_data?
100
+ @performance_data == true
101
+ end
102
+
103
+ def ldpaths_for_search
104
+ label_ldpath = search_config.results_label_ldpath
105
+ return {} if label_ldpath.blank?
106
+ ldpaths = { label: label_ldpath, uri: :subject_uri }
107
+ ldpaths[:altlabel] = search_config.results_altlabel_ldpath unless search_config.results_altlabel_ldpath.nil?
108
+ ldpaths[:id] = id_ldpath.present? ? id_ldpath : :subject_uri
109
+ ldpaths[:sort] = sort_ldpath.present? ? sort_ldpath : ldpaths[:label]
110
+ ldpaths
111
+ end
112
+
113
+ def id_ldpath
114
+ @id_ldpath ||= search_config.results_id_ldpath
115
+ end
116
+
117
+ def sort_ldpath
118
+ @sort_ldpath ||= search_config.results_sort_ldpath
119
+ end
120
+
65
121
  def preds_for_search
66
122
  label_pred_uri = search_config.results_label_predicate
67
- raise Qa::InvalidConfiguration, "required label_predicate is missing in search configuration for LOD authority #{auth_name}" if label_pred_uri.nil?
68
- preds = { label: label_pred_uri }
69
- preds[:uri] = :subject_uri
123
+ return {} if label_pred_uri.blank?
124
+ preds = { label: label_pred_uri, uri: :subject_uri }
70
125
  preds[:altlabel] = search_config.results_altlabel_predicate unless search_config.results_altlabel_predicate.nil?
71
126
  preds[:id] = id_predicate.present? ? id_predicate : :subject_uri
72
127
  preds[:sort] = sort_predicate.present? ? sort_predicate : preds[:label]
@@ -111,6 +166,14 @@ module Qa::Authorities
111
166
  lbl = '[' + lbl + ']' if labels.size > 1
112
167
  lbl
113
168
  end
169
+
170
+ def append_performance_data(results)
171
+ performance = { result_count: results.size,
172
+ fetch_time_s: access_time_s,
173
+ normalization_time_s: normalize_time_s,
174
+ total_time_s: (access_time_s + normalize_time_s) }
175
+ { performance: performance, results: results }
176
+ end
114
177
  end
115
178
  end
116
179
  end
@@ -37,5 +37,13 @@ module Qa
37
37
  def default_language
38
38
  @default_language ||= :en
39
39
  end
40
+
41
+ # When true, prevents ldpath requests from making additional network calls. All values will come from the context graph
42
+ # passed to the ldpath request.
43
+ attr_writer :limit_ldpath_to_context
44
+ def limit_ldpath_to_context?
45
+ return true if @limit_ldpath_to_context.nil?
46
+ @limit_ldpath_to_context
47
+ end
40
48
  end
41
49
  end
@@ -1,3 +1,3 @@
1
1
  module Qa
2
- VERSION = "4.0.0".freeze
2
+ VERSION = "4.1.0".freeze
3
3
  end
@@ -81,14 +81,54 @@ describe Qa::LinkedDataTermsController, type: :controller do
81
81
  end
82
82
 
83
83
  describe '#list' do
84
- let(:expected_results) { ['Auth1', 'Auth2', 'Auth3'] }
85
- before do
86
- allow(Qa::LinkedData::AuthorityService).to receive(:authority_names).and_return(expected_results)
84
+ context 'when details=false' do
85
+ let(:expected_results) { ['Auth1', 'Auth2', 'Auth3'] }
86
+ before do
87
+ allow(Qa::LinkedData::AuthorityService).to receive(:authority_names).and_return(expected_results)
88
+ end
89
+ it 'returns list of authorities' do
90
+ get :list
91
+ expect(response).to be_successful
92
+ expect(response.body).to eq expected_results.to_json
93
+ end
87
94
  end
88
- it 'returns list of authorities' do
89
- get :list
90
- expect(response).to be_successful
91
- expect(response.body).to eq expected_results.to_json
95
+
96
+ context 'when details=true' do
97
+ let(:expected_results) do
98
+ [
99
+ {
100
+ "label" => "oclc_fast term (QA)",
101
+ "uri" => "urn:qa:term:oclc_fast",
102
+ "authority" => "oclc_fast",
103
+ "action" => "term",
104
+ "language" => ["en"]
105
+ },
106
+ {
107
+ "label" => "oclc_fast search (QA)",
108
+ "uri" => "urn:qa:search:oclc_fast",
109
+ "authority" => "oclc_fast",
110
+ "action" => "search",
111
+ "language" => ["en"]
112
+ },
113
+ {
114
+ "label" => "oclc_fast search topic (QA)",
115
+ "uri" => "urn:qa:search:oclc_fast:topic",
116
+ "authority" => "oclc_fast",
117
+ "subauthority" => "topic",
118
+ "action" => "search",
119
+ "language" => ["en"]
120
+ }
121
+ ]
122
+ end
123
+
124
+ before do
125
+ allow(Qa::LinkedData::AuthorityService).to receive(:authority_details).and_return(expected_results)
126
+ end
127
+ it 'returns list of authorities' do
128
+ get :list, params: { details: 'true' }
129
+ expect(response).to be_successful
130
+ expect(response.body).to eq expected_results.to_json
131
+ end
92
132
  end
93
133
  end
94
134
 
@@ -254,6 +294,33 @@ describe Qa::LinkedDataTermsController, type: :controller do
254
294
  expect(results.first.key?('context')).to be false
255
295
  end
256
296
  end
297
+
298
+ context 'when requesting performance data' do
299
+ before do
300
+ Qa.config.disable_cors_headers
301
+ stub_request(:get, 'http://experimental.worldcat.org/fast/search?maximumRecords=3&query=cql.any%20all%20%22cornell%22&sortKeys=usage')
302
+ .to_return(status: 200, body: webmock_fixture('lod_oclc_all_query_3_results.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
303
+ end
304
+ it "returns basic data + performance data when performance_data='true'" do
305
+ get :search, params: { q: 'cornell', vocab: 'OCLC_FAST', maximumRecords: '3', performance_data: 'true' }
306
+ expect(response).to be_successful
307
+ results = JSON.parse(response.body)
308
+ expect(results).to be_kind_of Hash
309
+ expect(results.keys).to match_array ['performance', 'results']
310
+ expect(results['performance'].keys).to match_array ['result_count', 'fetch_time_s', 'normalization_time_s', 'total_time_s']
311
+ expect(results['performance']['total_time_s']).to eq results['performance']['fetch_time_s'] + results['performance']['normalization_time_s']
312
+ expect(results['performance']['result_count']).to eq 3
313
+ expect(results['results'].count).to eq 3
314
+ end
315
+
316
+ it "returns basic data only when performance_data='false'" do
317
+ get :search, params: { q: 'cornell', vocab: 'OCLC_FAST', maximumRecords: '3', performance_data: 'false' }
318
+ expect(response).to be_successful
319
+ results = JSON.parse(response.body)
320
+ expect(results).to be_kind_of Array
321
+ expect(results.size).to eq 3
322
+ end
323
+ end
257
324
  end
258
325
 
259
326
  describe '#show' do
@@ -378,48 +445,75 @@ describe Qa::LinkedDataTermsController, type: :controller do
378
445
  .to_return(status: 200, body: webmock_fixture('lod_loc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
379
446
  end
380
447
  it 'succeeds and defaults to json content type' do
381
- get :show, params: { id: 'sh85118553', vocab: 'LOC', subauthority: 'subjects' }
448
+ get :show, params: { id: 'sh 85118553', vocab: 'LOC', subauthority: 'subjects' }
382
449
  expect(response).to be_successful
383
450
  expect(response.content_type).to eq 'application/json'
384
451
  end
385
452
  end
386
453
  end
454
+
455
+ context 'when requesting performance data' do
456
+ before do
457
+ stub_request(:get, 'http://id.loc.gov/authorities/subjects/sh85118553')
458
+ .to_return(status: 200, body: webmock_fixture('lod_loc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
459
+ end
460
+ it "returns basic data + performance data when performance_data='true'" do
461
+ get :show, params: { id: 'sh 85118553', vocab: 'LOC', subauthority: 'subjects', performance_data: 'true' }
462
+ expect(response).to be_successful
463
+ results = JSON.parse(response.body)
464
+ expect(results).to be_kind_of Hash
465
+ expect(results.keys).to match_array ['performance', 'results']
466
+ expect(results['performance'].keys).to match_array ['predicate_count', 'fetch_time_s', 'normalization_time_s', 'total_time_s']
467
+ expect(results['performance']['total_time_s']).to eq results['performance']['fetch_time_s'] + results['performance']['normalization_time_s']
468
+ expect(results['performance']['predicate_count']).to eq 15
469
+ expect(results['results']['predicates'].count).to eq 15
470
+ end
471
+
472
+ it "returns basic data only when performance_data='false'" do
473
+ get :show, params: { id: 'sh 85118553', vocab: 'LOC', subauthority: 'subjects', performance_data: 'false' }
474
+ expect(response).to be_successful
475
+ results = JSON.parse(response.body)
476
+ expect(results).to be_kind_of Hash
477
+ expect(results.keys).not_to include('performance')
478
+ expect(results['predicates'].size).to eq 15
479
+ end
480
+ end
387
481
  end
388
482
 
389
483
  describe '#fetch' do
390
484
  context 'producing internal server error' do
391
485
  context 'when server returns 500' do
392
486
  before do
393
- stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369').to_return(status: 500)
487
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369').to_return(status: 500)
394
488
  end
395
489
  it 'returns 500' do
396
- expect(Rails.logger).to receive(:warn).with("Internal Server Error - Fetch term http://test.org/530369 unsuccessful for authority LOD_TERM_URI_PARAM_CONFIG")
397
- get :fetch, params: { vocab: 'LOD_TERM_URI_PARAM_CONFIG', uri: 'http://test.org/530369' }
490
+ expect(Rails.logger).to receive(:warn).with("Internal Server Error - Fetch term http://id.worldcat.org/fast/530369 unsuccessful for authority LOD_TERM_URI_PARAM_CONFIG")
491
+ get :fetch, params: { vocab: 'LOD_TERM_URI_PARAM_CONFIG', uri: 'http://id.worldcat.org/fast/530369' }
398
492
  expect(response.code).to eq('500')
399
493
  end
400
494
  end
401
495
 
402
496
  context 'when rdf format error' do
403
497
  before do
404
- stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369').to_return(status: 200)
498
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369').to_return(status: 200)
405
499
  allow(RDF::Graph).to receive(:load).and_raise(RDF::FormatError)
406
500
  end
407
501
  it 'returns 500' do
408
- msg = "RDF Format Error - Results from fetch term http://test.org/530369 for authority LOD_TERM_URI_PARAM_CONFIG was not identified as a valid RDF format. " \
502
+ msg = "RDF Format Error - Results from fetch term http://id.worldcat.org/fast/530369 for authority LOD_TERM_URI_PARAM_CONFIG was not identified as a valid RDF format. " \
409
503
  "You may need to include the linkeddata gem."
410
504
  expect(Rails.logger).to receive(:warn).with(msg)
411
- get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
505
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
412
506
  expect(response.code).to eq('500')
413
507
  end
414
508
  end
415
509
 
416
510
  context "when error isn't specifically handled" do
417
511
  before do
418
- stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369').to_return(status: 501)
512
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369').to_return(status: 501)
419
513
  end
420
514
  it 'returns 500' do
421
- expect(Rails.logger).to receive(:warn).with("Internal Server Error - Fetch term http://test.org/530369 unsuccessful for authority LOD_TERM_URI_PARAM_CONFIG")
422
- get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
515
+ expect(Rails.logger).to receive(:warn).with("Internal Server Error - Fetch term http://id.worldcat.org/fast/530369 unsuccessful for authority LOD_TERM_URI_PARAM_CONFIG")
516
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
423
517
  expect(response.code).to eq('500')
424
518
  end
425
519
  end
@@ -427,11 +521,11 @@ describe Qa::LinkedDataTermsController, type: :controller do
427
521
 
428
522
  context 'when service unavailable' do
429
523
  before do
430
- stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369').to_return(status: 503)
524
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369').to_return(status: 503)
431
525
  end
432
526
  it 'returns 503' do
433
- expect(Rails.logger).to receive(:warn).with("Service Unavailable - Fetch term http://test.org/530369 unsuccessful for authority LOD_TERM_URI_PARAM_CONFIG")
434
- get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
527
+ expect(Rails.logger).to receive(:warn).with("Service Unavailable - Fetch term http://id.worldcat.org/fast/530369 unsuccessful for authority LOD_TERM_URI_PARAM_CONFIG")
528
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
435
529
  expect(response.code).to eq('503')
436
530
  end
437
531
  end
@@ -450,19 +544,19 @@ describe Qa::LinkedDataTermsController, type: :controller do
450
544
  context 'in LOD_TERM_URI_PARAM_CONFIG authority' do
451
545
  context 'term found' do
452
546
  before do
453
- stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369')
547
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369')
454
548
  .to_return(status: 200, body: webmock_fixture('lod_oclc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
455
549
  end
456
550
 
457
551
  it 'succeeds and defaults to json content type' do
458
- get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
552
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
459
553
  expect(response).to be_successful
460
554
  expect(response.content_type).to eq 'application/json'
461
555
  end
462
556
 
463
557
  context 'and it was requested as json' do
464
558
  it 'succeeds and returns term data as json content type' do
465
- get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG', format: 'json' }
559
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG', format: 'json' }
466
560
  expect(response).to be_successful
467
561
  expect(response.content_type).to eq 'application/json'
468
562
  end
@@ -470,7 +564,7 @@ describe Qa::LinkedDataTermsController, type: :controller do
470
564
 
471
565
  context 'and it was requested as jsonld' do
472
566
  it 'succeeds and returns term data as jsonld content type' do
473
- get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG', format: 'jsonld' }
567
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG', format: 'jsonld' }
474
568
  expect(response).to be_successful
475
569
  expect(response.content_type).to eq 'application/ld+json'
476
570
  expect(JSON.parse(response.body).keys).to match_array ["@context", "@graph"]
@@ -479,11 +573,11 @@ describe Qa::LinkedDataTermsController, type: :controller do
479
573
 
480
574
  context 'blank nodes not included in predicates list' do
481
575
  before do
482
- stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369wbn')
576
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369wbn')
483
577
  .to_return(status: 200, body: webmock_fixture('lod_term_with_blanknode_objects.nt'), headers: { 'Content-Type' => 'application/n-triples' })
484
578
  end
485
579
  it 'succeeds' do
486
- get :fetch, params: { uri: 'http://test.org/530369wbn', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
580
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369wbn', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
487
581
  expect(response).to be_successful
488
582
  end
489
583
  end
@@ -492,11 +586,11 @@ describe Qa::LinkedDataTermsController, type: :controller do
492
586
  context 'when cors headers are enabled' do
493
587
  before do
494
588
  Qa.config.enable_cors_headers
495
- stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369')
589
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369')
496
590
  .to_return(status: 200, body: webmock_fixture('lod_oclc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
497
591
  end
498
592
  it 'Access-Control-Allow-Origin is *' do
499
- get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
593
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
500
594
  expect(response.headers['Access-Control-Allow-Origin']).to eq '*'
501
595
  end
502
596
  end
@@ -504,15 +598,42 @@ describe Qa::LinkedDataTermsController, type: :controller do
504
598
  context 'when cors headers are disabled' do
505
599
  before do
506
600
  Qa.config.disable_cors_headers
507
- stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369')
601
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369')
508
602
  .to_return(status: 200, body: webmock_fixture('lod_oclc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
509
603
  end
510
604
  it 'Access-Control-Allow-Origin is not present' do
511
- get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
605
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
512
606
  expect(response.headers.key?('Access-Control-Allow-Origin')).to be false
513
607
  end
514
608
  end
515
609
  end
610
+
611
+ context 'when requesting performance data' do
612
+ before do
613
+ stub_request(:get, 'http://localhost/test_default/term?uri=http://id.worldcat.org/fast/530369')
614
+ .to_return(status: 200, body: webmock_fixture('lod_oclc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
615
+ end
616
+ it "returns basic data + performance data when performance_data='true'" do
617
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG', performance_data: 'true' }
618
+ expect(response).to be_successful
619
+ results = JSON.parse(response.body)
620
+ expect(results).to be_kind_of Hash
621
+ expect(results.keys).to match_array ['performance', 'results']
622
+ expect(results['performance'].keys).to match_array ['predicate_count', 'fetch_time_s', 'normalization_time_s', 'total_time_s']
623
+ expect(results['performance']['total_time_s']).to eq results['performance']['fetch_time_s'] + results['performance']['normalization_time_s']
624
+ expect(results['performance']['predicate_count']).to eq 7
625
+ expect(results['results']['predicates'].count).to eq 7
626
+ end
627
+
628
+ it "returns basic data only when performance_data='false'" do
629
+ get :fetch, params: { uri: 'http://id.worldcat.org/fast/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG', performance_data: 'false' }
630
+ expect(response).to be_successful
631
+ results = JSON.parse(response.body)
632
+ expect(results).to be_kind_of Hash
633
+ expect(results.keys).not_to include('performance')
634
+ expect(results['predicates'].size).to eq 7
635
+ end
636
+ end
516
637
  end
517
638
 
518
639
  describe '#reload' do