qa 3.1.0 → 4.0.0.rc1

data/app/models/qa/linked_data/config/helper.rb

@@ -0,0 +1,34 @@
+# Provide helper method for common processing of configurations.
+module Qa
+  module LinkedData
+    module Config
+      class Helper
+        # Fetch a value from a hash map
+        def self.fetch(map, key, default)
+          map.fetch(key, default)
+        end
+
+        # Fetch a boolean value from a hash map throwing an exception if the value is not boolean
+        def self.fetch_boolean(map, key, default)
+          value = map.fetch(key, default)
+          raise Qa::InvalidConfiguration, "#{key} must be true or false" unless value == true || value == false
+          value
+        end
+
+        # Fetch a value from a hash map throwing an exception if the value is blank
+        def self.fetch_required(map, key, default)
+          value = map.fetch(key, default)
+          raise Qa::InvalidConfiguration, "#{key} is required" unless value
+          value
+        end
+
+        # Fetch a value from a hash map throwing an exception if the value is blank
+        def self.fetch_symbol(map, key, default)
+          value = map.fetch(key, default)
+          return value unless value.respond_to? :to_sym
+          value.to_sym
+        end
+      end
+    end
+  end
+end

data/app/services/qa/iri_template_service.rb

@@ -0,0 +1,31 @@
+# Provide service for building a URL based on an IRI Templated Link and its variable mappings based on provided substitutions.
+module Qa
+  class IriTemplateService
+    # Construct an url from an IriTemplate making identified substitutions
+    # @param url_config [Qa::IriTemplate::UrlConfig] configuration (json) holding the template and variable mappings
+    # @param substitutions [HashWithIndifferentAccess] name-value pairs to substitute into the url template
+    # @return [String] url with substitutions
+    def self.build_url(url_config:, substitutions:)
+      # TODO: This is a very simple approach using direct substitution into the template string.
+      #   Better would be to...
+      #     * patterns without a substitution are not included in the resulting URL
+      #     * appropriately adds '?' or '&'
+      #     * ensure proper escaping of values (e.g. value="A simple string" which is encoded as A%20simple%20string)
+      #   Even more advanced would be to...
+      #     * support BasicRepresentation (which is what it does now)
+      #     * support ExplicitRepresentation
+      #        * literal encoding for values (e.g. value="A simple string" becomes %22A%20simple%20string%22)
+      #        * language encoding for values (e.g. value="A simple string" becomes value="A simple string"@en which is encoded as %22A%20simple%20string%22%40en)
+      #        * type encoding for values (e.g. value=5.5 becomes value="5.5"^^http://www.w3.org/2001/XMLSchema#decimal which is encoded
+      #                                         as %225.5%22%5E%5Ehttp%3A%2F%2Fwww.w3.org%2F2001%2FXMLSchema%23decimal)
+      # Fuller implementations parse the template into component parts and then build the URL by adding parts in as applicable.
+      url = url_config.template
+      url_config.mapping.each do |m|
+        key = m.variable
+        url = url.gsub("{#{key}}", m.simple_value(substitutions[key]))
+        url = url.gsub("{?#{key}}", m.parameter_value(substitutions[key]))
+      end
+      url
+    end
+  end
+end

data/{lib/qa/authorities → app/services/qa}/linked_data/authority_service.rb

@@ -1,6 +1,5 @@
-# This module has the primary QA search method.  It also includes methods to process the linked data results and convert
-# them into the expected QA json results format.
-module Qa::Authorities
+# This module loads linked data authorities and provides access to their configurations.
+module Qa
   module LinkedData
     class AuthorityService
       # Load or reload the linked data configuration files
@@ -32,7 +31,7 @@ module Qa::Authorities
 
       # Get the configuration for an authority
       # @param [String] name of the authority
-      # @return [Array<String>] configuration for the specified authority
+      # @return [Hash] configuration for the specified authority
       def self.authority_config(authname)
         authority_configs[authname]
       end

data/app/services/qa/linked_data/authority_url_service.rb

@@ -0,0 +1,48 @@
+# Provide service for constructing the external access URL for an authority.
+module Qa
+  module LinkedData
+    class AuthorityUrlService
+      class << self
+        # Build a url for an authority/subauthority for the specified action.
+        # @param authority [Symbol] name of a registered authority
+        # @param subauthority [String] name of a subauthority
+        # @param action [Symbol] action with valid values :search or :term
+        # @param action_request [String] the request the user is making of the authority (e.g. query text or term id/uri)
+        # @param substitutions [Hash] variable-value pairs to substitute into the URL template
+        # @return a valid URL the submits the action request to the external authority
+        def build_url(action_config:, action:, action_request:, substitutions: {}, subauthority: nil)
+          action_validation(action)
+          url_config = action_config.url_config
+          selected_substitutions = url_config.extract_substitutions(combined_substitutions(action_config, action, action_request, substitutions, subauthority))
+          Qa::IriTemplateService.build_url(url_config: url_config, substitutions: selected_substitutions)
+        end
+
+        private
+
+          def action_validation(action)
+            return if [:search, :term].include? action
+            raise Qa::UnsupportedAction, "#{action} Not Supported - Action must be one of the supported actions (e.g. :term, :search)"
+          end
+
+          def combined_substitutions(action_config, action, action_request, substitutions, subauthority)
+            substitutions[action_request_variable(action_config, action)] = action_request
+            substitutions[action_subauth_variable(action_config)] = action_subauth_variable_value(action_config, subauthority) if subauthority.present?
+            substitutions
+          end
+
+          def action_request_variable(action_config, action)
+            key = action == :search ? :query : :term_id
+            action_config.qa_replacement_patterns[key]
+          end
+
+          def action_subauth_variable(action_config)
+            action_config.qa_replacement_patterns[:subauth]
+          end
+
+          def action_subauth_variable_value(action_config, subauthority)
+            action_config.subauthorities[subauthority.to_sym]
+          end
+      end
+    end
+  end
+end

data/app/services/qa/linked_data/deep_sort_service.rb

@@ -0,0 +1,238 @@
+# Provide service for for sorting an array of hash based on the values at a specified key in the hash.
+module Qa
+  module LinkedData
+    class DeepSortService
+      # @params [Array<Hash<Symbol,Array<RDF::Literal>>>] the array of hashes to sort
+      # @params [sort_key] the key in the hash on whose value the array will be sorted
+      # @return instance of this class
+      # @example the_array parameter
+      #   [
+      #     {:uri=>[#<RDF::URI:0x3fcff54a829c URI:http://id.loc.gov/authorities/names/n2010043281>],
+      #      :id=>[#<RDF::Literal:0x3fcff4a367b4("n 2010043281")>],
+      #      :label=>[#<RDF::Literal:0x3fcff54a9a98("Valli, Sabrina"@en)>],
+      #      :altlabel=>[],
+      #      :sort=>[#<RDF::Literal:0x3fcff54b4c18("2")>]},
+      #     {:uri=>[#<RDF::URI:0x3fcff54a829c URI:http://id.loc.gov/authorities/names/n201002344>],
+      #      :id=>[#<RDF::Literal:0x3fcff4a367b4("n 201002344")>],
+      #      :label=>[#<RDF::Literal:0x3fcff54a9a98("Cornell, Joseph"@en)>],
+      #      :altlabel=>[],
+      #      :sort=>[#<RDF::Literal:0x3fcff54b4c18("1")>]}
+      #   ]
+      def initialize(the_array, sort_key, preferred_language = nil)
+        @sortable_elements = the_array.map { |element| DeepSortElement.new(element, sort_key, preferred_language) }
+      end
+
+      # Sort an array of hash on the specified sort key.  The value in the hash at sort key is expected to be an array
+      # with one or more values that are RDF::Literals that translate to a number (e.g. 2), a string number (e.g. "3"),
+      # a string (e.g. "hello"), or a language qualified string (e.g. "hello"@en).
+      # The sort occurs in the following precedence.
+      # * preference for numeric sort (if only one value each and both are integers or a string that can be converted to an integer)
+      # * single value sort (if only one value each and at least one is not an integer)
+      # * multiple values sort (if either has multiple values)
+      # @return the sorted array
+      # @example returned sorted array
+      #   [
+      #     {:uri=>[#<RDF::URI:0x3fcff54a829c URI:http://id.loc.gov/authorities/names/n201002344>],
+      #      :id=>[#<RDF::Literal:0x3fcff4a367b4("n 201002344")>],
+      #      :label=>[#<RDF::Literal:0x3fcff54a9a98("Cornell, Joseph"@en)>],
+      #      :altlabel=>[],
+      #      :sort=>[#<RDF::Literal:0x3fcff54b4c18("1")>]},
+      #     {:uri=>[#<RDF::URI:0x3fcff54a829c URI:http://id.loc.gov/authorities/names/n2010043281>],
+      #      :id=>[#<RDF::Literal:0x3fcff4a367b4("n 2010043281")>],
+      #      :label=>[#<RDF::Literal:0x3fcff54a9a98("Valli, Sabrina"@en)>],
+      #      :altlabel=>[],
+      #      :sort=>[#<RDF::Literal:0x3fcff54b4c18("2")>]}
+      #   ]
+      def sort
+        @sortable_elements.sort.map(&:element)
+      end
+
+      class DeepSortElement
+        attr_reader :element, :literals, :preferred_language
+        private :preferred_language
+
+        delegate :size, to: :@literals
+
+        def initialize(element, sort_key, preferred_language)
+          element[sort_key] = Qa::LinkedData::LanguageSortService.new(element[sort_key], preferred_language).sort
+          @element = element
+          @literals = element[sort_key]
+          @preferred_language = preferred_language
+          @includes_preferred_language = includes_preferred_language?
+          @all_same_language = all_same_language?
+        end
+
+        def <=>(other)
+          return numeric_comparator(other) if integer? && other.integer?
+          return single_value_comparator(other) if single? && other.single?
+          multiple_value_comparator(other)
+        end
+
+        # @return true if there is a single literal that is an integer or a string that can be converted to an integer; otherwise, false
+        def integer?
+          return false unless single?
+          (/\A[-+]?\d+\z/ === literal.to_s) # rubocop:disable Style/CaseEquality
+        end
+
+        def integer(idx = 0)
+          Integer(literal(idx).to_s)
+        end
+
+        # @return true if there is only one value; otherwise, false
+        def single?
+          @single ||= literals.size == 1
+        end
+
+        def literal(idx = 0)
+          literals[idx]
+        end
+
+        def downcase_string(idx = 0)
+          to_downcase(literal(idx))
+        end
+
+        def language(lit = literals.first)
+          return nil unless Qa::LinkedData::LanguageService.literal_has_language_marker? lit
+          lit.language
+        end
+
+        def includes_preferred_language?
+          return @includes_preferred_language if @includes_preferred_language.present?
+          # literals are sorted by language with preferred language first in the list
+          @includes_preferred_language = (language == preferred_language)
+        end
+
+        def all_same_language?
+          return @all_same_language if @all_same_language.present?
+          # literals are sorted by language, so if first = last, then all are the same
+          @all_same_language = (language(literals.first) == language(literals.last))
+        end
+
+        def languages
+          filtered_literals_by_language.keys
+        end
+
+        def filtered_literals(filter_language)
+          filtered_literals_by_language.fetch(filter_language, [])
+        end
+
+        private
+
+          # If both test values are single value and both are integers, do a numeric sort
+          def numeric_comparator(other)
+            integer <=> other.integer
+          end
+
+          # If both test values are single value and at least one is not numeric, do a string sort taking language into consideration
+          # * sort values if neither has a language marker or they both have the same language marker
+          # * otherwise, sort language markers
+          def single_value_comparator(other)
+            return downcase_string <=> other.downcase_string if same_language?(literal, other.literal)
+            compare_languages(language, other.language)
+          end
+
+          def compare_languages(lang, other_lang)
+            return -1 if preferred_language? lang
+            return 1 if preferred_language? other_lang
+            return -1 if other_lang.blank?
+            return 1 if lang.blank?
+            lang <=> other_lang
+          end
+
+          # If at least one of the test values has multiple values, sort the multiple values taking language into consideration
+          # * if both lists have all the same language or no language markers at all, just sort the lists and compare each element
+          # * if either list has the preferred language, try to sort the two lists by element after filtering for the preferred language
+          # * otherwise, sort by language until there is a difference
+          def multiple_value_comparator(other)
+            return single_language_list_comparator(other) if all_same_language? && other.all_same_language?
+            return specified_language_list_comparator(other, preferred_language) if includes_preferred_language? && other.includes_preferred_language?
+            multi_language_list_comparator(other)
+          end
+
+          def single_language_list_comparator(other)
+            list_comparator(literals, other.literals)
+          end
+
+          def specified_language_list_comparator(other, lang)
+            filtered = filtered_literals(lang)
+            other_filtered = other.filtered_literals(lang)
+            return -1 if !filtered.empty? && other_filtered.empty?
+            return 1 if filtered.empty? && !other_filtered.empty?
+            list_comparator(filtered, other_filtered)
+          end
+
+          # Walk through language sorted lists
+          # * for each language, determine how closely the list of terms matches
+          # * prioritize the list that gets the most low values
+          def multi_language_list_comparator(other)
+            combined_languages = languages.concat(other.languages).uniq
+            by_language_comparisons = {}
+            combined_languages.each do |lang|
+              cmp = list_comparator(filtered_literals(lang), other.filtered_literals(lang))
+              by_language_comparisons[lang] = cmp
+            end
+            cmp_sum = by_language_comparisons.values.sum
+            return 1 if cmp_sum.positive?
+            return -1 if cmp_sum.negative?
+            0
+          end
+
+          def list_comparator(list, other_list)
+            # if an element doesn't have any terms in a language, the other element sorts lower
+            return -1 if other_list.empty?
+            return 1 if list.empty?
+            shorter_list_size = [list.size, other_list.size].min
+            cmp = 0
+            0.upto(shorter_list_size - 1) do |idx|
+              cmp = to_downcase(list[idx]) <=> to_downcase(other_list[idx])
+              return cmp unless cmp.zero?
+            end
+            return cmp if list.size == other_list.size
+            other_list.size < list.size ? 1 : -1 # didn't find any diffs, shorter list is considered lower
+          end
+
+          def same_language?(lit, other_lit)
+            return false if only_one_has_language_marker?(lit, other_lit)
+            return true if neither_have_language_markers?(lit, other_lit)
+            lit.language == other_lit.language
+          end
+
+          def neither_have_language_markers?(lit, other_lit)
+            !language?(lit) && !language?(other_lit)
+          end
+
+          def only_one_has_language_marker?(lit, other_lit)
+            (!language?(lit) && language?(other_lit)) || (language?(lit) && !language?(other_lit))
+          end
+
+          def language?(lit)
+            Qa::LinkedData::LanguageService.literal_has_language_marker? lit
+          end
+
+          def preferred_language?(lang)
+            preferred_language.present? ? lang == preferred_language : false
+          end
+
+          def to_downcase(lit)
+            lit.to_s.downcase
+          end
+
+          def filtered_literals_by_language
+            @filtered_literals_by_language ||= create_all_filters
+          end
+
+          def create_all_filters
+            bins = {}
+            0.upto(size - 1) do |idx|
+              lang = language(literals[idx])
+              filter = bins.fetch(lang, [])
+              filter << literal(idx)
+              bins[lang] = filter
+            end
+            bins
+          end
+      end
+      private_constant :DeepSortElement
+    end
+  end
+end

data/app/services/qa/linked_data/graph_service.rb

@@ -0,0 +1,106 @@
+# Extend the RDF graph to include additional processing methods.
+module Qa
+  module LinkedData
+    class GraphService
+      class << self
+        # Retrieve linked data from specified url
+        # @param [String] url from which to retrieve linked data
+        # @return [RDF::Graph] graph of linked data
+        def load_graph(url:)
+          RDF::Graph.load(url)
+        rescue IOError => e
+          process_error(e, url)
+        end
+
+        # Create a new graph with statements filtered out
+        # @param graph [RDF::Graph] the original graph to be filtered
+        # @param language [Array<Symbol>] will keep any statement whose object's language matches the language filter
+        #          (only applies to statements that respond to language) (e.g. [:fr] or [:en, :fr])
+        # @param remove_blanknode_subjects [Boolean] will remove any statement whose subject is a blanknode, if true
+        # @return [RDF::Graph] a new instance of graph with statements not matching the filters removed
+        def filter(graph:, language: nil, remove_blanknode_subjects: false)
+          return graph unless graph.present?
+          return graph unless language.present? || remove_blanknode_subjects
+          filtered_graph = deep_copy(graph: graph)
+          filtered_graph.statements.each do |st|
+            filtered_graph.delete(st) if filter_out_blanknode(remove_blanknode_subjects, st.subject) || filter_out_language(graph, language, st)
+          end
+          filtered_graph
+        end
+
+        # Get object values from the graph that have the subject-predicate.
+        # @param graph [RDF::Graph] the graph to search
+        # @param subject [RDF::URI] the URI of the subject
+        # @param predicate [RDF::URI] the URI of the predicate
+        # @return [Array] all object values for the subject-predicate pair
+        def object_values(graph:, subject:, predicate:)
+          values = []
+          graph.query([subject, predicate, :object]) do |statement|
+            values << statement.object
+          end
+          values
+        end
+
+        def deep_copy(graph:)
+          new_graph = RDF::Graph.new
+          graph.statements.each do |st|
+            new_graph.insert(st.dup)
+          end
+          new_graph
+        end
+
+        private
+
+          def filter_out_blanknode(remove, subj)
+            remove && subj.anonymous?
+          end
+
+          # Filter out language based on...
+          # * do not remove if the object literal does not respond to :language
+          # * do not remove if the object literal does not have a language marker
+          # * do not remove if the object has the targeted language marker
+          # * do not remove if none of the other objects with this statement's predicate have the targeted language marker
+          def filter_out_language(graph, language, statement)
+            return false if language.blank?
+            return false unless Qa::LinkedData::LanguageService.literal_has_language_marker?(statement.object)
+            objects = object_values(graph: graph, subject: statement.subject, predicate: statement.predicate)
+            return false unless at_least_one_object_has_language?(objects, language)
+            !language.include?(statement.object.language)
+          end
+
+          def at_least_one_object_has_language?(objects, language)
+            objects.each do |obj|
+              next unless Qa::LinkedData::LanguageService.literal_has_language_marker?(obj)
+              next unless language.include? obj.language
+              return true
+            end
+            false
+          end
+
+          def process_error(e, url)
+            uri = URI(url)
+            raise RDF::FormatError, "Unknown RDF format of results returned by #{uri}. (RDF::FormatError)  You may need to include gem 'linkeddata'." if e.is_a? RDF::FormatError
+            response_code = ioerror_code(e)
+            case response_code
+            when '404'
+              raise Qa::TermNotFound, "#{uri} Not Found - Term may not exist at LOD Authority. (HTTPNotFound - 404)"
+            when '500'
+              raise Qa::ServiceError, "#{uri.hostname} on port #{uri.port} is not responding.  Try again later. (HTTPServerError - 500)"
+            when '503'
+              raise Qa::ServiceUnavailable, "#{uri.hostname} on port #{uri.port} is not responding.  Try again later. (HTTPServiceUnavailable - 503)"
+            else
+              raise Qa::ServiceError, "Unknown error for #{uri.hostname} on port #{uri.port}.  Try again later. (Cause - #{e.message})"
+            end
+          end
+
+          def ioerror_code(e)
+            msg = e.message
+            return 'format' if msg.start_with? "Unknown RDF format"
+            a = msg.size - 4
+            z = msg.size - 2
+            msg[a..z]
+          end
+      end
+    end
+  end
+end