qa 3.1.0 → 4.0.0.rc1

data/lib/qa/authorities/linked_data/generic_authority.rb

@@ -10,7 +10,8 @@ module Qa::Authorities
     # @see Qa::LinkedDataTermsController#show
     # @see Qa::Authorities::LinkedData::Config
     class GenericAuthority < Base
-      attr_reader :auth_config
+      attr_reader :authority_config
+      private :authority_config
 
       delegate :supports_term?, :term_subauthorities?, :term_subauthority?,
                :term_id_expects_uri?, :term_id_expects_id?, to: :term_config
@@ -19,7 +20,7 @@ module Qa::Authorities
       delegate :subauthority?, :subauthorities?, to: :search_config, prefix: 'search'
 
       def initialize(auth_name)
-        @auth_config = Qa::Authorities::LinkedData::Config.new(auth_name)
+        @authority_config = Qa::Authorities::LinkedData::Config.new(auth_name)
       end
 
       def reload_authorities
@@ -27,7 +28,7 @@ module Qa::Authorities
       end
 
       def authorities_service
-        @authorities_service ||= Qa::Authorities::LinkedData::AuthorityService
+        @authorities_service ||= Qa::LinkedData::AuthorityService
       end
 
       def search_service
@@ -45,11 +46,11 @@ module Qa::Authorities
       private
 
         def search_config
-          auth_config.search
+          authority_config.search
         end
 
         def term_config
-          auth_config.term
+          authority_config.term
         end
     end
   end

data/lib/qa/authorities/linked_data/rdf_helper.rb

@@ -7,6 +7,9 @@ module Qa::Authorities
     module RdfHelper
       private
 
+        # TODO: elr - The bulk of the methods in this class moved to app/services/linked_data/rdf_service.rb.  The remaining
+        # methods are expected to move in a later refactor.
+
         def object_value(stmt_hash, consolidated_hash, name, as_string = true)
           new_object_value = stmt_hash[name]
           new_object_value = new_object_value.to_s if as_string
@@ -24,70 +27,6 @@ module Qa::Authorities
           consolidated_hash
         end
 
-        def get_linked_data(url)
-          begin
-            graph = RDF::Graph.load(url)
-          rescue IOError => e
-            process_error(e, url)
-          end
-          graph
-        end
-
-        def process_error(e, url)
-          uri = URI(url)
-          raise RDF::FormatError, "Unknown RDF format of results returned by #{uri}. (RDF::FormatError)  You may need to include gem 'linkeddata'." if e.is_a? RDF::FormatError
-          response_code = ioerror_code(e)
-          case response_code
-          when '404'
-            raise Qa::TermNotFound, "#{uri} Not Found - Term may not exist at LOD Authority. (HTTPNotFound - 404)"
-          when '500'
-            raise Qa::ServiceError, "#{uri.hostname} on port #{uri.port} is not responding.  Try again later. (HTTPServerError - 500)"
-          when '503'
-            raise Qa::ServiceUnavailable, "#{uri.hostname} on port #{uri.port} is not responding.  Try again later. (HTTPServiceUnavailable - 503)"
-          else
-            raise Qa::ServiceError, "Unknown error for #{uri.hostname} on port #{uri.port}.  Try again later. (Cause - #{e.message})"
-          end
-        end
-
-        def ioerror_code(e)
-          msg = e.message
-          return 'format' if msg.start_with? "Unknown RDF format"
-          a = msg.size - 4
-          z = msg.size - 2
-          msg[a..z]
-        end
-
-        # Filter a graph to the specified languages
-        # @param [RDF::Graph] the graph to be filtered.
-        # @param [String | Symbol | Array<String|Symbol>] language for filtering graph (e.g. "en" or :en or ["en", "fr"] or [:en, :fr])
-        # @returns [RDF::Graph] graph of linked data filtered on the specified languages
-        def filter_language(graph, language)
-          language = normalize_language(language)
-          return graph if language.nil?
-          graph.each do |st|
-            graph.delete(st) unless !st.object.respond_to?(:language) || st.object.language.nil? || language.include?(st.object.language)
-          end
-          graph
-        end
-
-        # Filter a graph to remove any statement with a blanknode for the subject
-        # @param [RDF::Graph] the graph to be filtered.
-        # @returns [RDF::Graph] graph of linked data with blanknodes removed
-        def filter_out_blanknodes(graph)
-          return graph if graph.subjects.blank?
-          graph.each do |st|
-            graph.delete(st) if st.subject.anonymous?
-          end
-          graph
-        end
-
-        def normalize_language(language)
-          language = [language.to_sym] if language.is_a? String
-          language = [language] if language.is_a? Symbol
-          return nil unless language.is_a? Array
-          language
-        end
-
         def extract_preds(graph, preds)
           RDF::Query.execute(graph) do
             preds[:required].each do |key, pred|
@@ -100,17 +39,11 @@ module Qa::Authorities
         end
 
         def sort_string_by_language(str_literals)
-          return str_literals if str_literals.nil? || str_literals.size <= 0
-          str_literals.sort! { |a, b| a.language <=> b.language }
-          str_literals.collect!(&:to_s)
-          str_literals.uniq!
+          return str_literals if str_literals.blank?
+          str_literals = Qa::LinkedData::LanguageSortService.new(str_literals).sort
+          str_literals.map!(&:to_s).uniq!
           str_literals.delete_if { |s| s.nil? || s.length <= 0 }
         end
-
-        def blank_node?(obj)
-          return true if obj.to_s.starts_with? "_:g"
-          false
-        end
     end
   end
 end

data/lib/qa/authorities/linked_data/search_query.rb

@@ -3,98 +3,100 @@
 module Qa::Authorities
   module LinkedData
     class SearchQuery
-      include Qa::Authorities::LinkedData::RdfHelper
+      class_attribute :authority_service, :graph_service, :language_service, :language_sort_service, :results_mapper_service
+      self.authority_service = Qa::LinkedData::AuthorityUrlService
+      self.graph_service = Qa::LinkedData::GraphService
+      self.language_service = Qa::LinkedData::LanguageService
+      self.language_sort_service = Qa::LinkedData::LanguageSortService
+      self.results_mapper_service = Qa::LinkedData::Mapper::SearchResultsMapperService
 
       # @param [SearchConfig] search_config The search portion of the config
       def initialize(search_config)
         @search_config = search_config
       end
 
-      attr_reader :search_config
+      attr_reader :search_config, :graph, :language
+      private :graph, :language
 
       delegate :subauthority?, :supports_sort?, to: :search_config
 
       # Search a linked data authority
-      # @param [String] the query
-      # @param [Symbol] (optional) language: language used to select literals when multi-language is supported (e.g. :en, :fr, etc.)
-      # @param [Hash] (optional) replacements: replacement values with { pattern_name (defined in YAML config) => value }
-      # @param [String] subauth: the subauthority to query
+      # @praram [String] the query
+      # @param language [Symbol] (optional) language used to select literals when multi-language is supported (e.g. :en, :fr, etc.)
+      # @param replacements [Hash] (optional) replacement values with { pattern_name (defined in YAML config) => value }
+      # @param subauth [String] (optional) the subauthority to query
+      # @param context [Boolean] (optional) true if context should be returned with the results; otherwise, false (default: false)
       # @return [String] json results
       # @example Json Results for Linked Data Search
       #   [ {"uri":"http://id.worldcat.org/fast/5140","id":"5140","label":"Cornell, Joseph"},
       #     {"uri":"http://id.worldcat.org/fast/72456","id":"72456","label":"Cornell, Sarah Maria, 1802-1832"},
       #     {"uri":"http://id.worldcat.org/fast/409667","id":"409667","label":"Cornell, Ezra, 1807-1874"} ]
-      def search(query, language: nil, replacements: {}, subauth: nil)
+      def search(query, language: nil, replacements: {}, subauth: nil, context: false)
         raise Qa::InvalidLinkedDataAuthority, "Unable to initialize linked data search sub-authority #{subauth}" unless subauth.nil? || subauthority?(subauth)
-        language ||= search_config.language
-        url = search_config.url_with_replacements(query, subauth, replacements)
+        @context = context
+        @language = language_service.preferred_language(user_language: language, authority_language: search_config.language)
+        url = authority_service.build_url(action_config: search_config, action: :search, action_request: query, substitutions: replacements, subauthority: subauth)
         Rails.logger.info "QA Linked Data search url: #{url}"
-        graph = get_linked_data(url)
-        parse_search_authority_response(graph, language)
+        load_graph(url: url)
+        parse_search_authority_response
       end
 
       private
 
-        def parse_search_authority_response(graph, language)
-          graph = filter_language(graph, language) unless language.nil?
-          graph = filter_out_blanknodes(graph)
-          results = extract_preds(graph, preds_for_search)
-          consolidated_results = consolidate_search_results(results)
-          json_results = convert_search_to_json(consolidated_results)
-          sort_search_results(json_results)
+        def load_graph(url:)
+          @graph = graph_service.load_graph(url: url)
+          @graph = graph_service.filter(graph: @graph, language: language, remove_blanknode_subjects: true)
         end
 
-        def preds_for_search
-          { required: required_search_preds, optional: optional_search_preds }
+        def parse_search_authority_response
+          results = results_mapper_service.map_values(graph: @graph, predicate_map: preds_for_search, sort_key: :sort,
+                                                      preferred_language: @language, context_map: context_map)
+          convert_results_to_json(results)
         end
 
-        def required_search_preds
-          label_pred_uri = search_config.results_label_predicate
-          raise Qa::InvalidConfiguration, "required label_predicate is missing in search configuration for LOD authority #{auth_name}" if label_pred_uri.nil?
-          { label: label_pred_uri }
+        def context_map
+          context? ? search_config.context_map : nil
         end
 
-        def optional_search_preds
-          preds = {}
+        def context?
+          @context == true
+        end
+
+        def preds_for_search
+          label_pred_uri = search_config.results_label_predicate
+          raise Qa::InvalidConfiguration, "required label_predicate is missing in search configuration for LOD authority #{auth_name}" if label_pred_uri.nil?
+          preds = { label: label_pred_uri }
+          preds[:uri] = :subject_uri
           preds[:altlabel] = search_config.results_altlabel_predicate unless search_config.results_altlabel_predicate.nil?
           preds[:id] = search_config.results_id_predicate unless search_config.results_id_predicate.nil?
-          preds[:sort] = search_config.results_sort_predicate unless search_config.results_sort_predicate.nil?
+          preds[:sort] = sort_predicate.present? ? sort_predicate : preds[:label]
           preds
         end
 
-        def consolidate_search_results(results) # rubocop:disable Metrics/MethodLength
-          consolidated_results = {}
-          return consolidated_results if results.nil? || !results.count.positive?
-          results.each do |statement|
-            stmt_hash = statement.to_h
-            uri = stmt_hash[:uri].to_s
-            consolidated_hash = init_consolidated_hash(consolidated_results, uri, stmt_hash[:id].to_s)
-
-            consolidated_hash[:label] = object_value(stmt_hash, consolidated_hash, :label, false)
-            consolidated_hash[:altlabel] = object_value(stmt_hash, consolidated_hash, :altlabel, false)
-            consolidated_hash[:sort] = object_value(stmt_hash, consolidated_hash, :sort, false)
-            consolidated_results[uri] = consolidated_hash
-          end
-          consolidated_results.each do |res|
-            consolidated_hash = res[1]
-            consolidated_hash[:label] = sort_string_by_language consolidated_hash[:label]
-            consolidated_hash[:altlabel] = sort_string_by_language consolidated_hash[:altlabel]
-            consolidated_hash[:sort] = sort_string_by_language consolidated_hash[:sort]
-          end
-          consolidated_results
+        def sort_predicate
+          @sort_predicate ||= search_config.results_sort_predicate
         end
 
-        def convert_search_to_json(consolidated_results)
+        def convert_results_to_json(results)
           json_results = []
-          consolidated_results.each do |uri, h|
-            json_results << { uri: uri, id: h[:id], label: full_label(h[:label], h[:altlabel]), sort: h[:sort] }
-          end
+          results.each { |result| json_results << convert_result_to_json(result) }
           json_results
         end
 
+        def convert_result_to_json(result)
+          json_result = {}
+          json_result[:uri] = result[:uri].first.to_s
+          json_result[:id] = result[:id].first.to_s
+          json_result[:label] = full_label(result[:label], result[:altlabel])
+          json_result[:context] = result[:context] if context?
+          json_result
+        end
+
         def full_label(label = [], altlabel = [])
+          label = language_sort_service.new(label, language).sort
+          altlabel = language_sort_service.new(altlabel, language).sort
           lbl = wrap_labels(label)
-          lbl += " (#{altlabel.join(', ')})" unless altlabel.nil? || altlabel.length <= 0
+          lbl += " (#{altlabel.join(', ')})" if altlabel.present?
           lbl = lbl.slice(0..95) + '...' if lbl.length > 98
           lbl.strip
         end
@@ -105,55 +107,6 @@ module Qa::Authorities
           lbl = '[' + lbl + ']' if labels.size > 1
           lbl
         end
-
-        def sort_search_results(json_results) # rubocop:disable Metrics/MethodLength
-          return json_results unless supports_sort?
-          json_results.sort! do |a, b|
-            cmp = sort_when_missing_sort_predicate(a, b)
-            next cmp unless cmp.nil?
-
-            cmp = numeric_sort(a, b)
-            next cmp unless cmp.nil?
-
-            as = a[:sort].collect(&:downcase)
-            bs = b[:sort].collect(&:downcase)
-            cmp = 0
-            0.upto([as.size, bs.size].max - 1) do |i|
-              cmp = sort_when_same_but_one_has_more_values(as, bs, i)
-              break unless cmp.nil?
-
-              cmp = (as[i] <=> bs[i])
-              break if cmp.nonzero? # stop checking as soon as a value in the two lists are different
-            end
-            cmp
-          end
-          json_results.each { |h| h.delete(:sort) }
-        end
-
-        def sort_when_missing_sort_predicate(a, b)
-          return 0 unless a.key?(:sort) || b.key?(:sort) # leave unchanged if both are missing
-          return -1 unless a.key? :sort # consider missing a value lower than existing b value
-          return 1 unless b.key? :sort # consider missing b value lower than existing a value
-          nil
-        end
-
-        def sort_when_same_but_one_has_more_values(as, bs, current_list_size)
-          return -1 if as.size <= current_list_size # consider shorter a list of values lower then longer b list
-          return 1 if bs.size <= current_list_size # consider shorter b list of values lower then longer a list
-          nil
-        end
-
-        def numeric_sort(a, b)
-          return nil if a[:sort].size > 1
-          return nil if b[:sort].size > 1
-          return nil unless s_is_i? a[:sort][0]
-          return nil unless s_is_i? b[:sort][0]
-          Integer(a[:sort][0]) <=> Integer(b[:sort][0])
-        end
-
-        def s_is_i?(s)
-          /\A[-+]?\d+\z/ === s # rubocop:disable Style/CaseEquality
-        end
     end
   end
 end

data/lib/qa/authorities/loc/generic_authority.rb

@@ -24,9 +24,9 @@ module Qa::Authorities
     end
 
     def build_query_url(q)
-      escaped_query = URI.escape(q)
-      authority_fragment = Loc.get_url_for_authority(subauthority) + URI.escape(subauthority)
-      "https://id.loc.gov/search/?q=#{escaped_query}&q=#{authority_fragment}&format=json"
+      escaped_query = ERB::Util.url_encode(q)
+      authority_fragment = Loc.get_url_for_authority(subauthority) + ERB::Util.url_encode(subauthority)
+      "http://id.loc.gov/search/?q=#{escaped_query}&q=#{authority_fragment}&format=json"
     end
 
     def find(id)
@@ -34,7 +34,7 @@ module Qa::Authorities
     end
 
     def find_url(id)
-      "https://id.loc.gov/authorities/#{@subauthority}/#{id}.json"
+      "http://id.loc.gov/authorities/#{@subauthority}/#{id}.json"
     end
 
     private

data/lib/qa/authorities/web_service_base.rb

@@ -14,18 +14,11 @@ module Qa::Authorities
     # @param url [String]
     # @return [Hash] a parsed JSON response
     def json(url)
+      Rails.logger.info "Retrieving json for url: #{url}"
       r = response(url).body
       JSON.parse(r)
     end
 
-    ##
-    # @deprecated Use #json instead
-    def get_json(url)
-      warn '[DEPRECATED] #get_json is deprecated; use #json instead.' \
-           "Called from #{Gem.location_of_caller.join(':')}."
-      json(url)
-    end
-
     ##
     # Make a web request and retrieve the response.
     #

data/lib/qa/configuration.rb

@@ -30,5 +30,12 @@ module Qa
 
     # Hold linked data authority configs
     attr_accessor :linked_data_authority_configs
+
+    # For linked data access, specify default language for sorting and selection.  The default is only used if a language is not
+    # specified in the authority's configuration file and not passed in as a parameter.  (e.g. :en, [:en], or [:en, :fr])
+    attr_writer :default_language
+    def default_language
+      @default_language ||= :en
+    end
   end
 end

data/lib/qa/version.rb

@@ -1,3 +1,3 @@
 module Qa
-  VERSION = "3.1.0".freeze
+  VERSION = "4.0.0.rc1".freeze
 end

data/lib/tasks/mesh.rake

@@ -1,26 +1,27 @@
 require 'benchmark'
 
-namespace :mesh do
-  desc "Import MeSH terms from the file $MESH_FILE, it will update any terms which are already in the database"
-  # task :import => :environment do
-  task import: :environment do
-    fname = ENV['MESH_FILE']
-    if fname.nil?
-      puts "Need to set $MESH_FILE with path to file to ingest"
-      next # transfers control out of this block
-    end
-    Benchmark.bm(30) do |bm|
-      bm.report("Importing #{fname}") do
-        m = Qa::Authorities::MeshTools::MeshImporter.new
-        File.open(fname) do |f|
-          m.import_from_file(f)
+namespace :qa do
+  namespace :mesh do
+    desc "Import MeSH terms from the file $MESH_FILE, it will update any terms which are already in the database"
+    task import: :environment do
+      fname = ENV['MESH_FILE']
+      if fname.nil?
+        puts "Need to set $MESH_FILE with path to file to ingest"
+        next # transfers control out of this block
+      end
+      Benchmark.bm(30) do |bm|
+        bm.report("Importing #{fname}") do
+          m = Qa::Authorities::MeshTools::MeshImporter.new
+          File.open(fname) do |f|
+            m.import_from_file(f)
+          end
         end
       end
     end
-  end
 
-  desc "Delete all mesh terms from the database---not implemented"
-  task :clear do
-    puts "Not implemented"
+    desc "Delete all mesh terms from the database---not implemented"
+    task :clear do
+      puts "Not implemented"
+    end
   end
 end