qa 2.0.0 → 2.3.0

data/lib/qa/authorities/getty/ulan.rb

@@ -11,29 +11,26 @@ module Qa::Authorities
       "http://vocab.getty.edu/sparql.json?query=#{URI.escape(sparql(q)).gsub('&', '%26')}&_implicit=false&implicit=true&_equivalent=false&_form=%2Fsparql"
     end
 
-    def sparql(q)
+    def sparql(q) # rubocop:disable Metrics/MethodLength
       search = untaint(q)
       # if more than one term is supplied, check both preferred and alt labels
       if search.include?(' ')
-        ex = "("
-        search.split(' ').each do |i|
-          ex += "regex(CONCAT(?name, ' ', ?alt), \"#{i}\",\"i\" ) && "
+        clauses = search.split(' ').collect do |i|
+          %((regex(?name, "#{i}", "i") || regex(?alt, "#{i}", "i")))
         end
-        ex = ex[0..ex.length - 4]
-        ex += ")"
+        ex = "(#{clauses.join(' && ')})"
       else
-        ex = "regex(?name, \"#{search}\", \"i\")"
+        ex = %(regex(?name, "#{search}", "i"))
       end
       # The full text index matches on fields besides the term, so we filter to ensure the match is in the term.
-      sparql = "SELECT DISTINCT ?s ?name ?bio {
-              ?s a skos:Concept; luc:term \"#{search}\";
-                 skos:inScheme <http://vocab.getty.edu/ulan/> ;
-                 gvp:prefLabelGVP [skosxl:literalForm ?name] ;
-                 foaf:focus/gvp:biographyPreferred [schema:description ?bio] ;
-                 skos:altLabel ?alt .
-              FILTER #{ex} .
-            } ORDER BY ?name"
-      sparql
+      %(SELECT DISTINCT ?s ?name ?bio {
+        ?s a skos:Concept; luc:term "#{search}";
+            skos:inScheme <http://vocab.getty.edu/ulan/> ;
+            gvp:prefLabelGVP [skosxl:literalForm ?name] ;
+            foaf:focus/gvp:biographyPreferred [schema:description ?bio] ;
+            skos:altLabel ?alt .
+        FILTER #{ex} .
+      } ORDER BY ?name).gsub(/[\s\n]+/, " ")
     end
 
     def untaint(q)

data/lib/qa/authorities/linked_data.rb

@@ -3,6 +3,7 @@ module Qa::Authorities
     extend ActiveSupport::Autoload
     autoload :GenericAuthority
     autoload :RdfHelper
+    autoload :AuthorityService
     autoload :SearchQuery
     autoload :FindTerm
     autoload :Config

data/lib/qa/authorities/linked_data/authority_service.rb

@@ -0,0 +1,47 @@
+# This module has the primary QA search method.  It also includes methods to process the linked data results and convert
+# them into the expected QA json results format.
+module Qa::Authorities
+  module LinkedData
+    class AuthorityService
+      # Load or reload the linked data configuration files
+      def self.load_authorities
+        auth_cfg = {}
+        # load QA configured linked data authorities
+        Dir[File.join(Qa::Engine.root, 'config', 'authorities', 'linked_data', '*.json')].each do |fn|
+          auth = File.basename(fn, '.json').upcase.to_sym
+          json = File.read(File.expand_path(fn, __FILE__))
+          cfg = JSON.parse(json).deep_symbolize_keys
+          auth_cfg[auth] = cfg
+        end
+
+        # load app configured linked data authorities and overrides
+        Dir[Rails.root.join('config', 'authorities', 'linked_data', '*.json')].each do |fn|
+          auth = File.basename(fn, '.json').upcase.to_sym
+          json = File.read(File.expand_path(fn, __FILE__))
+          cfg = JSON.parse(json).deep_symbolize_keys
+          auth_cfg[auth] = cfg
+        end
+        Qa.config.linked_data_authority_configs = auth_cfg
+      end
+
+      # Get the list of names of the loaded authorities
+      # @return [Array<String>] all loaded authority configurations
+      def self.authority_configs
+        Qa.config.linked_data_authority_configs
+      end
+
+      # Get the configuration for an authority
+      # @param [String] name of the authority
+      # @return [Array<String>] configuration for the specified authority
+      def self.authority_config(authname)
+        authority_configs[authname]
+      end
+
+      # Get the list of names of the loaded authorities
+      # @return [Array<String>] names of the authority config files that are currently loaded
+      def self.authority_names
+        authority_configs.keys.sort
+      end
+    end
+  end
+end

data/lib/qa/authorities/linked_data/config.rb

@@ -1,6 +1,7 @@
 require 'qa/authorities/linked_data/config/term_config'.freeze
 require 'qa/authorities/linked_data/config/search_config'.freeze
 require 'json'
+require 'erb'
 
 # Provide attr_reader methods for linked data authority configurations.  Some default configurations are provided for several
 # linked data authorities and can be found at /config/authorities/linked_data.  You can add configurations for new authorities by
@@ -17,6 +18,10 @@ require 'json'
 module Qa::Authorities
   module LinkedData
     class Config
+      class << self
+        include ERB::Util
+      end
+
       attr_reader :authority_name
       attr_reader :authority_config
 
@@ -39,7 +44,7 @@ module Qa::Authorities
       # Return the full configuration for an authority
       # @return [String] the authority configuration
       def auth_config
-        @authority_config ||= LINKED_DATA_AUTHORITIES_CONFIG[@authority_name]
+        @authority_config ||= Qa::Authorities::LinkedData::AuthorityService.authority_config(@authority_name)
         raise Qa::InvalidLinkedDataAuthority, "Unable to initialize linked data authority '#{@authority_name}'" if @authority_config.nil?
         @authority_config
       end
@@ -56,7 +61,8 @@ module Qa::Authorities
         pred_uri
       end
 
-      def self.replace_pattern(url, pattern, value)
+      def self.replace_pattern(url, pattern, value, encode = false)
+        value = url_encode(value).gsub(".", "%2E") if encode
         url.gsub("{?#{pattern}}", value)
       end
 
@@ -71,7 +77,7 @@ module Qa::Authorities
         config.each do |param_key, rep_pattern|
           s_param_key = param_key.to_s
           value = replacements[param_key] || replacements[s_param_key] || rep_pattern[:default]
-          url = replace_pattern(url, param_key, value)
+          url = replace_pattern(url, param_key, value, rep_pattern[:encode])
         end
         url
       end

data/lib/qa/authorities/linked_data/config/search_config.rb

@@ -95,12 +95,26 @@ module Qa::Authorities
         search_config.fetch(:qa_replacement_patterns)
       end
 
+      # Should the replacement pattern be encoded?
+      # @return [Boolean] true, if the pattern should be encoded; otherwise, false
+      def qa_replacement_encoded?(pattern_key)
+        map_key = qa_replacement_patterns[pattern_key].to_sym
+        replacement_encoded? map_key
+      end
+
       # Are there replacement parameters configured for search query?
       # @return [True|False] true if there are replacement parameters configured for search query; otherwise, false
       def replacements?
         replacement_count.positive?
       end
 
+      # Should the replacement parameter be encoded?
+      # @return [True|False] true if the replacement parameter should be encoded; otherwise, false
+      def replacement_encoded?(map_key)
+        return false unless url_mappings[map_key].key? :encode
+        url_mappings[map_key][:encode]
+      end
+
       # Return the number of possible replacement values to make in the search URL
       # @return [Integer] the configured number of possible replacements in the search url
       def replacement_count
@@ -159,8 +173,8 @@ module Qa::Authorities
       # @return [String] the search encoded url
       def url_with_replacements(query, sub_auth = nil, search_replacements = {})
         return nil unless supports_search?
-        sub_auth = sub_auth.to_sym if sub_auth.is_a? String
-        url = Config.replace_pattern(url_template, qa_replacement_patterns[:query], query)
+        sub_auth = sub_auth.to_sym if sub_auth.present?
+        url = Config.replace_pattern(url_template, qa_replacement_patterns[:query], query, qa_replacement_encoded?(:query))
         url = Config.process_subauthority(url, subauthority_replacement_pattern, subauthorities, sub_auth) if subauthorities?
         url = Config.apply_replacements(url, replacements, search_replacements) if replacements?
         url

data/lib/qa/authorities/linked_data/config/term_config.rb

@@ -112,12 +112,26 @@ module Qa::Authorities
         Config.config_value(term_config, :qa_replacement_patterns)
       end
 
+      # Should the replacement pattern be encoded?
+      # @return [Boolean] true, if the pattern should be encoded; otherwise, false
+      def term_qa_replacement_encoded?(pattern_key)
+        map_key = term_qa_replacement_patterns[pattern_key].to_sym
+        term_replacement_encoded? map_key
+      end
+
       # Are there replacement parameters configured for term fetch?
-      # @return [True|False] true if there are replacement parameters configured for term fetch; otherwise, false
+      # @return [Boolean] true if there are replacement parameters configured for term fetch; otherwise, false
       def term_replacements?
         term_replacement_count.positive?
       end
 
+      # Should the replacement parameter be encoded?
+      # @return [Boolean] true if the replacement parameter should be encoded; otherwise, false
+      def term_replacement_encoded?(map_key)
+        return false unless term_url_mappings[map_key].key? :encode
+        term_url_mappings[map_key][:encode]
+      end
+
       # Return the number of possible replacement values to make in the term URL
       # @return [Integer] the configured number of possible replacements in the term url
       def term_replacement_count
@@ -129,7 +143,7 @@ module Qa::Authorities
       def term_replacements
         return @term_replacements unless @term_replacements.nil?
         @term_replacements = {}
-        @term_replacements = term_url_mappings.select { |k, _v| !term_qa_replacement_patterns.include?(k) } unless term_config.nil? || term_url_mappings.nil?
+        @term_replacements = term_url_mappings.select { |k, _v| !term_qa_replacement_patterns.value?(k.to_s) } unless term_config.nil? || term_url_mappings.nil?
         @term_replacements
       end
 
@@ -176,7 +190,7 @@ module Qa::Authorities
       def term_url_with_replacements(id, sub_auth = nil, replacements = {})
         return nil unless supports_term?
         sub_auth = sub_auth.to_sym if sub_auth.is_a? String
-        url = Config.replace_pattern(term_url_template, term_qa_replacement_patterns[:term_id], id)
+        url = Config.replace_pattern(term_url_template, term_qa_replacement_patterns[:term_id], id, term_qa_replacement_encoded?(:term_id))
         url = Config.process_subauthority(url, term_subauthority_replacement_pattern, term_subauthorities, sub_auth) if term_subauthorities?
         url = Config.apply_replacements(url, term_replacements, replacements) if term_replacements?
         url

data/lib/qa/authorities/linked_data/find_term.rb

@@ -33,13 +33,14 @@ module Qa::Authorities
       #     "http://schema.org/name":["Cornell University","Ithaca (N.Y.). Cornell University"],
       #     "http://www.w3.org/2004/02/skos/core#altLabel":["Ithaca (N.Y.). Cornell University"],
       #     "http://schema.org/sameAs":["http://id.loc.gov/authorities/names/n79021621","https://viaf.org/viaf/126293486"] } }
-      def find(id, language: nil, replacements: {}, subauth: nil)
+      def find(id, language: nil, replacements: {}, subauth: nil, jsonld: false)
         raise Qa::InvalidLinkedDataAuthority, "Unable to initialize linked data term sub-authority #{subauth}" unless subauth.nil? || term_subauthority?(subauth)
         language ||= term_config.term_language
         url = term_config.term_url_with_replacements(id, subauth, replacements)
         Rails.logger.info "QA Linked Data term url: #{url}"
         graph = get_linked_data(url)
         return "{}" unless graph.size.positive?
+        return graph.dump(:jsonld, standard_prefixes: true) if jsonld
         parse_term_authority_response(id, graph, language)
       end
 
@@ -76,7 +77,7 @@ module Qa::Authorities
           preds
         end
 
-        def consolidate_term_results(results)
+        def consolidate_term_results(results) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize # TODO: Explore a way to simplify
           consolidated_results = {}
           results.each do |statement|
             stmt_hash = statement.to_h
@@ -123,7 +124,7 @@ module Qa::Authorities
           json_results.first
         end
 
-        def predicates_with_subject_uri(graph, expected_uri)
+        def predicates_with_subject_uri(graph, expected_uri) # rubocop:disable Metrics/MethodLength
           predicates_hash = {}
           graph.statements.each do |st|
             subj = st.subject.to_s

data/lib/qa/authorities/linked_data/generic_authority.rb

@@ -22,7 +22,13 @@ module Qa::Authorities
         @auth_config = Qa::Authorities::LinkedData::Config.new(auth_name)
       end
 
-      include WebServiceBase
+      def reload_authorities
+        @authorities_service.load_authorities
+      end
+
+      def authorities_service
+        @authorities_service ||= Qa::Authorities::LinkedData::AuthorityService
+      end
 
       def search_service
         @search_service ||= Qa::Authorities::LinkedData::SearchQuery.new(search_config)
@@ -34,6 +40,7 @@ module Qa::Authorities
 
       delegate :search, to: :search_service
       delegate :find, to: :item_service
+      delegate :load_authorities, :authority_names, to: :authorities_service
 
       private
 

data/lib/qa/authorities/linked_data/rdf_helper.rb

@@ -57,6 +57,10 @@ module Qa::Authorities
           msg[a..z]
         end
 
+        # Filter a graph to the specified languages
+        # @param [RDF::Graph] the graph to be filtered.
+        # @param [String | Symbol | Array<String|Symbol>] language for filtering graph (e.g. "en" or :en or ["en", "fr"] or [:en, :fr])
+        # @returns [RDF::Graph] graph of linked data filtered on the specified languages
         def filter_language(graph, language)
           language = normalize_language(language)
           return graph if language.nil?
@@ -66,6 +70,17 @@ module Qa::Authorities
           graph
         end
 
+        # Filter a graph to remove any statement with a blanknode for the subject
+        # @param [RDF::Graph] the graph to be filtered.
+        # @returns [RDF::Graph] graph of linked data with blanknodes removed
+        def filter_out_blanknodes(graph)
+          return graph if graph.subjects.blank?
+          graph.each do |st|
+            graph.delete(st) if st.subject.anonymous?
+          end
+          graph
+        end
+
         def normalize_language(language)
           language = [language.to_sym] if language.is_a? String
           language = [language] if language.is_a? Symbol

data/lib/qa/authorities/linked_data/search_query.rb

@@ -37,6 +37,7 @@ module Qa::Authorities
 
         def parse_search_authority_response(graph, language)
           graph = filter_language(graph, language) unless language.nil?
+          graph = filter_out_blanknodes(graph)
           results = extract_preds(graph, preds_for_search)
           consolidated_results = consolidate_search_results(results)
           json_results = convert_search_to_json(consolidated_results)
@@ -61,7 +62,7 @@ module Qa::Authorities
           preds
         end
 
-        def consolidate_search_results(results)
+        def consolidate_search_results(results) # rubocop:disable Metrics/MethodLength
           consolidated_results = {}
           return consolidated_results if results.nil? || !results.count.positive?
           results.each do |statement|
@@ -105,11 +106,14 @@ module Qa::Authorities
           lbl
         end
 
-        def sort_search_results(json_results)
+        def sort_search_results(json_results) # rubocop:disable Metrics/MethodLength
           return json_results unless supports_sort?
           json_results.sort! do |a, b|
             cmp = sort_when_missing_sort_predicate(a, b)
-            next unless cmp.nil?
+            next cmp unless cmp.nil?
+
+            cmp = numeric_sort(a, b)
+            next cmp unless cmp.nil?
 
             as = a[:sort].collect(&:downcase)
             bs = b[:sort].collect(&:downcase)
@@ -138,6 +142,18 @@ module Qa::Authorities
           return 1 if bs.size <= current_list_size # consider shorter b list of values lower then longer a list
           nil
         end
+
+        def numeric_sort(a, b)
+          return nil if a[:sort].size > 1
+          return nil if b[:sort].size > 1
+          return nil unless s_is_i? a[:sort][0]
+          return nil unless s_is_i? b[:sort][0]
+          Integer(a[:sort][0]) <=> Integer(b[:sort][0])
+        end
+
+        def s_is_i?(s)
+          /\A[-+]?\d+\z/ === s # rubocop:disable Style/CaseEquality
+        end
     end
   end
 end

data/lib/qa/authorities/loc/generic_authority.rb

@@ -26,7 +26,7 @@ module Qa::Authorities
     def build_query_url(q)
       escaped_query = URI.escape(q)
       authority_fragment = Loc.get_url_for_authority(subauthority) + URI.escape(subauthority)
-      "http://id.loc.gov/search/?q=#{escaped_query}&q=#{authority_fragment}&format=json"
+      "https://id.loc.gov/search/?q=#{escaped_query}&q=#{authority_fragment}&format=json"
     end
 
     def find(id)
@@ -34,7 +34,7 @@ module Qa::Authorities
     end
 
     def find_url(id)
-      "http://id.loc.gov/authorities/#{@subauthority}/#{id}.json"
+      "https://id.loc.gov/authorities/#{@subauthority}/#{id}.json"
     end
 
     private

data/lib/qa/authorities/loc_subauthority.rb

@@ -18,7 +18,7 @@ module Qa::Authorities::LocSubauthority
     ]
   end
 
-  def vocabularies
+  def vocabularies # rubocop:disable Metrics/MethodLength
     [
       "graphicMaterials",
       "organizations",
@@ -40,7 +40,7 @@ module Qa::Authorities::LocSubauthority
     ["edtf"]
   end
 
-  def preservation
+  def preservation # rubocop:disable Metrics/MethodLength
     [
       "contentLocationType",
       "copyrightStatus",

data/lib/qa/authorities/local.rb

@@ -20,15 +20,13 @@ module Qa::Authorities
         if config[:local_path].starts_with?(File::Separator)
           config[:local_path]
         else
-          File.join(Rails.root, config[:local_path])
+          Rails.root.join(config[:local_path]).to_s # TODO: Rails.root.join returns class Pathname, which may be ok.  Added to_s because of failing regression test.
         end
       end
 
       # Local sub-authorities are any YAML files in the subauthorities_path
       def names
-        unless Dir.exist? subauthorities_path
-          raise Qa::ConfigDirectoryNotFound, "There's no directory at #{subauthorities_path}. You must create it in order to use local authorities"
-        end
+        raise Qa::ConfigDirectoryNotFound, "There's no directory at #{subauthorities_path}. You must create it in order to use local authorities" unless Dir.exist? subauthorities_path
         Dir.entries(subauthorities_path).map { |f| File.basename(f, ".yml") if f =~ /yml$/ }.compact
       end
 

data/lib/qa/authorities/local/file_based_authority.rb

@@ -25,7 +25,7 @@ module Qa::Authorities
     private
 
       def terms
-        subauthority_hash = YAML.load(File.read(subauthority_filename))
+        subauthority_hash = YAML.load(File.read(subauthority_filename)) # rubocop:disable Security/YAMLLoad # TODO: Explore how to change this to safe_load.  Many tests fail when making this change.
         terms = subauthority_hash.with_indifferent_access.fetch(:terms, [])
         normalize_terms(terms)
       end

data/lib/qa/authorities/local/mysql_table_based_authority.rb

@@ -5,8 +5,7 @@ module Qa
         self.table_index = "index_qa_local_authority_entries_on_lower_label_and_authority"
 
         def self.check_for_index
-          conn = ActiveRecord::Base.connection
-          if table_or_view_exists? && conn.index_name_exists?(table_name.to_sym, table_index, :default).blank?
+          if table_or_view_exists? && index_name_exists? # rubocop:disable Style/GuardClause
             Rails.logger.error "You've installed mysql local authority tables, but you haven't indexed the lower label. "
             "Rails doesn't support functional indexes in migrations, so we tried to execute it for you but something went wrong...\n" \
             "Make sure your table has a lower_label column, which is virtually created, and that the column is indexed." \
@@ -19,6 +18,16 @@ module Qa
           return [] if q.blank?
           output_set(base_relation.where('lower_label like ?', "#{q.downcase}%").limit(25))
         end
+
+        def self.index_name_exists?
+          conn = ActiveRecord::Base.connection
+          if ActiveRecord::VERSION::MAJOR >= 5 && ActiveRecord::VERSION::MINOR >= 1
+            conn.index_name_exists?(table_name, table_index).blank?
+          else
+            conn.index_name_exists?(table_name, table_index, :default).blank?
+          end
+        end
+        private_class_method :index_name_exists?
       end
     end
   end