RubyGems - qa - Versions diffs - 2.1.2 → 2.2.0 - Mend

qa 2.1.2 → 2.2.0

Files changed (28) hide show

checksums.yaml +4 -4
data/README.md +22 -3
data/app/controllers/qa/linked_data_terms_controller.rb +108 -49
data/config/initializers/linked_data_authorities.rb +1 -17
data/config/routes.rb +3 -0
data/lib/generators/qa/apidoc/templates/public/qa/apidoc/apidoc.json +598 -29
data/lib/generators/qa/apidoc/templates/public/qa/apidoc/index.html +1 -1
data/lib/generators/qa/install/templates/config/initializers/qa.rb +6 -0
data/lib/qa/authorities/getty/aat.rb +13 -6
data/lib/qa/authorities/getty/tgn.rb +10 -21
data/lib/qa/authorities/getty/ulan.rb +12 -15
data/lib/qa/authorities/linked_data.rb +1 -0
data/lib/qa/authorities/linked_data/authority_service.rb +47 -0
data/lib/qa/authorities/linked_data/config.rb +1 -1
data/lib/qa/authorities/linked_data/find_term.rb +2 -1
data/lib/qa/authorities/linked_data/generic_authority.rb +8 -1
data/lib/qa/configuration.rb +18 -0
data/lib/qa/version.rb +1 -1
data/spec/controllers/linked_data_terms_controller_spec.rb +180 -2
data/spec/fixtures/authorities/linked_data/lod_term_uri_param_config.json +27 -0
data/spec/lib/authorities/getty/aat_spec.rb +14 -9
data/spec/lib/authorities/getty/tgn_spec.rb +6 -17
data/spec/lib/authorities/getty/ulan_spec.rb +5 -5
data/spec/lib/authorities/linked_data/authority_service_spec.rb +47 -0
data/spec/lib/configuration_spec.rb +58 -0
metadata +9 -4
data/config/authorities/linked_data/agrovoc_ld4l_cache.json +0 -85
data/config/authorities/linked_data/geonames_ld4l_cache.json +0 -102

data/lib/generators/qa/apidoc/templates/public/qa/apidoc/index.html CHANGED

@@ -3,7 +3,7 @@
 <html lang="en">
 <head>
   <meta charset="UTF-8">
-  <title>QA 2.1 Linked Data API</title>
+  <title>QA 2.2 Linked Data API</title>
     <link rel="stylesheet" type="text/css" href="./swagger-ui.css" >
     <link rel="icon" type="image/png" href="./favicon-32x32.png" sizes="32x32" />
     <link rel="icon" type="image/png" href="./favicon-16x16.png" sizes="16x16" />

data/lib/generators/qa/install/templates/config/initializers/qa.rb CHANGED

@@ -4,4 +4,10 @@ Qa.config do |config|
   # More information on CORS headers at: https://fetch.spec.whatwg.org/#cors-protocol
   # config.enable_cors_headers
   # config.disable_cors_headers
+  # Provide a token that allows reloading of linked data authorities through the controller
+  # action '/reload/linked_data/authorities?auth_token=YOUR_AUTH_TOKEN_DEFINED_HERE' without
+  # requiring a restart of rails. By default, reloading through the browser is not allowed
+  # when the token is nil or blank.  Change to any string to control who has access to reload.
+  # config.authorized_reload_token = YOUR_AUTH_TOKEN_DEFINED_HERE
 end

data/lib/qa/authorities/getty/aat.rb CHANGED

@@ -10,16 +10,23 @@ module Qa::Authorities
       "http://vocab.getty.edu/sparql.json?query=#{URI.escape(sparql(q))}&_implicit=false&implicit=true&_equivalent=false&_form=%2Fsparql"
     end
-    def sparql(q)
+    def sparql(q) # rubocop:disable Metrics/MethodLength
       search = untaint(q)
+      if search.include?(' ')
+        clauses = search.split(' ').collect do |i|
+          %((regex(?name, "#{i}", "i")))
+        end
+        ex = "(#{clauses.join(' && ')})"
+      else
+        ex = %(regex(?name, "#{search}", "i"))
+      end
       # The full text index matches on fields besides the term, so we filter to ensure the match is in the term.
-      sparql = "SELECT ?s ?name {
-              ?s a skos:Concept; luc:term \"#{search}\";
+      %(SELECT ?s ?name {
+              ?s a skos:Concept; luc:term "#{search}";
                  skos:inScheme <http://vocab.getty.edu/aat/> ;
                  gvp:prefLabelGVP [skosxl:literalForm ?name].
-              FILTER regex(?name, \"#{search}\", \"i\") .
-            } ORDER BY ?name"
-      sparql
+              FILTER #{ex} .
+            } ORDER BY ?name).gsub(/[\s\n]+/, " ")
     end
     def untaint(q)

data/lib/qa/authorities/getty/tgn.rb CHANGED

@@ -19,31 +19,20 @@ module Qa::Authorities
     def sparql(q) # rubocop:disable Metrics/MethodLength
       search = untaint(q)
       if search.include?(' ')
-        ex = "(("
-        search.split(' ').each do |i|
-          ex += "regex(CONCAT(?name, ', ', REPLACE(str(?par), \",[^,]+,[^,]+$\", \"\")), \"#{i}\",\"i\" ) && "
+        clauses = search.split(' ').collect do |i|
+          %((regex(?name, "#{i}", "i") || regex(?alt, "#{i}", "i")))
         end
-        ex = ex[0..ex.length - 4]
-        ex += ') && ('
-        search.split(' ').each do |i|
-          ex += "regex(?name, \"#{i}\",\"i\" ) || "
-        end
-        ex = ex[0..ex.length - 4]
-        ex += ") )"
+        ex = "(#{clauses.join(' && ')})"
       else
-        ex = "regex(?name, \"#{search}\", \"i\")"
+        ex = %(regex(?name, "#{search}", "i"))
       end
-      # The full text index matches on fields besides the term, so we filter to ensure the match is in the term.
-      sparql = "SELECT DISTINCT ?s ?name ?par {
-              ?s a skos:Concept; luc:term \"#{search}\";
-                 skos:inScheme <http://vocab.getty.edu/tgn/> ;
-                 gvp:prefLabelGVP [skosxl:literalForm ?name] ;
+      %(SELECT DISTINCT ?s ?name ?par {
+        ?s a skos:Concept; luc:term "#{search}";
+            skos:inScheme <http://vocab.getty.edu/ulan/> ;
+            gvp:prefLabelGVP [skosxl:literalForm ?name] ;
                   gvp:parentString ?par .
-              FILTER #{ex} .
-            } ORDER BY ?name ASC(?par)"
-      sparql
+        FILTER #{ex} .
+      } ORDER BY ?name ASC(?par)).gsub(/[\s\n]+/, " ")
     end
     def untaint(q)

data/lib/qa/authorities/getty/ulan.rb CHANGED

@@ -15,25 +15,22 @@ module Qa::Authorities
       search = untaint(q)
       # if more than one term is supplied, check both preferred and alt labels
       if search.include?(' ')
-        ex = "("
-        search.split(' ').each do |i|
-          ex += "regex(CONCAT(?name, ' ', ?alt), \"#{i}\",\"i\" ) && "
+        clauses = search.split(' ').collect do |i|
+          %((regex(?name, "#{i}", "i") || regex(?alt, "#{i}", "i")))
         end
-        ex = ex[0..ex.length - 4]
-        ex += ")"
+        ex = "(#{clauses.join(' && ')})"
       else
-        ex = "regex(?name, \"#{search}\", \"i\")"
+        ex = %(regex(?name, "#{search}", "i"))
       end
       # The full text index matches on fields besides the term, so we filter to ensure the match is in the term.
-      sparql = "SELECT DISTINCT ?s ?name ?bio {
-              ?s a skos:Concept; luc:term \"#{search}\";
-                 skos:inScheme <http://vocab.getty.edu/ulan/> ;
-                 gvp:prefLabelGVP [skosxl:literalForm ?name] ;
-                 foaf:focus/gvp:biographyPreferred [schema:description ?bio] ;
-                 skos:altLabel ?alt .
-              FILTER #{ex} .
-            } ORDER BY ?name"
-      sparql
+      %(SELECT DISTINCT ?s ?name ?bio {
+        ?s a skos:Concept; luc:term "#{search}";
+            skos:inScheme <http://vocab.getty.edu/ulan/> ;
+            gvp:prefLabelGVP [skosxl:literalForm ?name] ;
+            foaf:focus/gvp:biographyPreferred [schema:description ?bio] ;
+            skos:altLabel ?alt .
+        FILTER #{ex} .
+      } ORDER BY ?name).gsub(/[\s\n]+/, " ")
     end
     def untaint(q)

data/lib/qa/authorities/linked_data.rb CHANGED

@@ -3,6 +3,7 @@ module Qa::Authorities
     extend ActiveSupport::Autoload
     autoload :GenericAuthority
     autoload :RdfHelper
+    autoload :AuthorityService
     autoload :SearchQuery
     autoload :FindTerm
     autoload :Config

data/lib/qa/authorities/linked_data/authority_service.rb ADDED

@@ -0,0 +1,47 @@
+# This module has the primary QA search method.  It also includes methods to process the linked data results and convert
+# them into the expected QA json results format.
+module Qa::Authorities
+  module LinkedData
+    class AuthorityService
+      # Load or reload the linked data configuration files
+      def self.load_authorities
+        auth_cfg = {}
+        # load QA configured linked data authorities
+        Dir[File.join(Qa::Engine.root, 'config', 'authorities', 'linked_data', '*.json')].each do |fn|
+          auth = File.basename(fn, '.json').upcase.to_sym
+          json = File.read(File.expand_path(fn, __FILE__))
+          cfg = JSON.parse(json).deep_symbolize_keys
+          auth_cfg[auth] = cfg
+        end
+        # load app configured linked data authorities and overrides
+        Dir[Rails.root.join('config', 'authorities', 'linked_data', '*.json')].each do |fn|
+          auth = File.basename(fn, '.json').upcase.to_sym
+          json = File.read(File.expand_path(fn, __FILE__))
+          cfg = JSON.parse(json).deep_symbolize_keys
+          auth_cfg[auth] = cfg
+        end
+        Qa.config.linked_data_authority_configs = auth_cfg
+      end
+      # Get the list of names of the loaded authorities
+      # @return [Array<String>] all loaded authority configurations
+      def self.authority_configs
+        Qa.config.linked_data_authority_configs
+      end
+      # Get the configuration for an authority
+      # @param [String] name of the authority
+      # @return [Array<String>] configuration for the specified authority
+      def self.authority_config(authname)
+        authority_configs[authname]
+      end
+      # Get the list of names of the loaded authorities
+      # @return [Array<String>] names of the authority config files that are currently loaded
+      def self.authority_names
+        authority_configs.keys.sort
+      end
+    end
+  end
+end

data/lib/qa/authorities/linked_data/config.rb CHANGED

@@ -44,7 +44,7 @@ module Qa::Authorities
       # Return the full configuration for an authority
       # @return [String] the authority configuration
       def auth_config
-        @authority_config ||= LINKED_DATA_AUTHORITIES_CONFIG[@authority_name]
+        @authority_config ||= Qa::Authorities::LinkedData::AuthorityService.authority_config(@authority_name)
         raise Qa::InvalidLinkedDataAuthority, "Unable to initialize linked data authority '#{@authority_name}'" if @authority_config.nil?
         @authority_config
       end

data/lib/qa/authorities/linked_data/find_term.rb CHANGED

@@ -33,13 +33,14 @@ module Qa::Authorities
       #     "http://schema.org/name":["Cornell University","Ithaca (N.Y.). Cornell University"],
       #     "http://www.w3.org/2004/02/skos/core#altLabel":["Ithaca (N.Y.). Cornell University"],
       #     "http://schema.org/sameAs":["http://id.loc.gov/authorities/names/n79021621","https://viaf.org/viaf/126293486"] } }
-      def find(id, language: nil, replacements: {}, subauth: nil)
+      def find(id, language: nil, replacements: {}, subauth: nil, jsonld: false)
         raise Qa::InvalidLinkedDataAuthority, "Unable to initialize linked data term sub-authority #{subauth}" unless subauth.nil? || term_subauthority?(subauth)
         language ||= term_config.term_language
         url = term_config.term_url_with_replacements(id, subauth, replacements)
         Rails.logger.info "QA Linked Data term url: #{url}"
         graph = get_linked_data(url)
         return "{}" unless graph.size.positive?
+        return graph.dump(:jsonld, standard_prefixes: true) if jsonld
         parse_term_authority_response(id, graph, language)
       end

data/lib/qa/authorities/linked_data/generic_authority.rb CHANGED

@@ -22,7 +22,13 @@ module Qa::Authorities
         @auth_config = Qa::Authorities::LinkedData::Config.new(auth_name)
       end
-      include WebServiceBase
+      def reload_authorities
+        @authorities_service.load_authorities
+      end
+      def authorities_service
+        @authorities_service ||= Qa::Authorities::LinkedData::AuthorityService
+      end
       def search_service
         @search_service ||= Qa::Authorities::LinkedData::SearchQuery.new(search_config)
@@ -34,6 +40,7 @@ module Qa::Authorities
       delegate :search, to: :search_service
       delegate :find, to: :item_service
+      delegate :load_authorities, :authority_names, to: :authorities_service
       private

data/lib/qa/configuration.rb CHANGED

@@ -12,5 +12,23 @@ module Qa
     def disable_cors_headers
       @cors_headers_enabled = false
     end
+    # Provide a token that allows reloading of linked data authorities through the controller
+    # action '/reload/linked_data/authorities?auth_token=YOUR_AUTH_TOKEN_DEFINED_HERE' without
+    # requiring a restart of rails. By default, reloading through the browser is not allowed
+    # when the token is nil or blank.  Change to your approved token string in
+    # config/initializers/qa.rb.
+    attr_writer :authorized_reload_token
+    def authorized_reload_token
+      @authorized_reload_token ||= nil
+    end
+    def valid_authority_reload_token?(token)
+      return false if token.blank? || authorized_reload_token.blank?
+      token == authorized_reload_token
+    end
+    # Hold linked data authority configs
+    attr_accessor :linked_data_authority_configs
   end
 end

data/lib/qa/version.rb CHANGED

@@ -1,3 +1,3 @@
 module Qa
-  VERSION = "2.1.2".freeze
+  VERSION = "2.2.0".freeze
 end

data/spec/controllers/linked_data_terms_controller_spec.rb CHANGED

@@ -61,6 +61,14 @@ describe Qa::LinkedDataTermsController, type: :controller do
     end
   end
+  describe '#check_uri_param' do
+    it 'returns 400 if the uri is missing' do
+      expect(Rails.logger).to receive(:warn).with("Required fetch param 'uri' is missing or empty")
+      get :fetch, params: { uri: '', vocab: 'OCLC_FAST' }
+      expect(response.code).to eq('400')
+    end
+  end
   describe '#init_authority' do
     context 'when the authority does not exist' do
       it 'returns 400' do
@@ -71,6 +79,18 @@ describe Qa::LinkedDataTermsController, type: :controller do
     end
   end
+  describe '#list' do
+    let(:expected_results) { ['Auth1', 'Auth2', 'Auth3'] }
+    before do
+      allow(Qa::Authorities::LinkedData::AuthorityService).to receive(:authority_names).and_return(expected_results)
+    end
+    it 'returns list of authorities' do
+      get :list
+      expect(response).to be_successful
+      expect(response.body).to eq expected_results.to_json
+    end
+  end
   describe '#search' do
     context 'producing internal server error' do
       context 'when server returns 500' do
@@ -324,9 +344,28 @@ describe Qa::LinkedDataTermsController, type: :controller do
           stub_request(:get, 'http://artemide.art.uniroma2.it:8081/agrovoc/rest/v1/data?uri=http://aims.fao.org/aos/agrovoc/c_9513')
             .to_return(status: 200, body: webmock_fixture('lod_agrovoc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
         end
-        it 'succeeds' do
+        it 'succeeds and defaults to json content type' do
           get :show, params: { id: 'c_9513', vocab: 'AGROVOC' }
           expect(response).to be_successful
+          expect(response.content_type).to eq 'application/json'
+        end
+        context 'and it was requested as json' do
+          it 'succeeds and returns term data as json content type' do
+            get :show, params: { id: 'c_9513', vocab: 'AGROVOC', format: 'json' }
+            expect(response).to be_successful
+            expect(response.content_type).to eq 'application/json'
+          end
+        end
+        context 'and it was requested as jsonld' do
+          it 'succeeds and returns term data as jsonld content type' do
+            get :show, params: { id: 'c_9513', vocab: 'AGROVOC', format: 'jsonld' }
+            expect(response).to be_successful
+            expect(response.content_type).to eq 'application/ld+json'
+            expect(JSON.parse(response.body).keys).to match_array ["@context", "@graph"]
+          end
         end
       end
     end
@@ -337,11 +376,150 @@ describe Qa::LinkedDataTermsController, type: :controller do
           stub_request(:get, 'http://id.loc.gov/authorities/subjects/sh85118553')
             .to_return(status: 200, body: webmock_fixture('lod_loc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
         end
-        it 'succeeds' do
+        it 'succeeds and defaults to json content type' do
           get :show, params: { id: 'sh85118553', vocab: 'LOC', subauthority: 'subjects' }
           expect(response).to be_successful
+          expect(response.content_type).to eq 'application/json'
+        end
+      end
+    end
+  end
+  describe '#fetch' do
+    context 'producing internal server error' do
+      context 'when server returns 500' do
+        before do
+          stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369').to_return(status: 500)
+        end
+        it 'returns 500' do
+          expect(Rails.logger).to receive(:warn).with("Internal Server Error - Fetch term http://test.org/530369 unsuccessful for authority LOD_TERM_URI_PARAM_CONFIG")
+          get :fetch, params: { vocab: 'LOD_TERM_URI_PARAM_CONFIG', uri: 'http://test.org/530369' }
+          expect(response.code).to eq('500')
         end
       end
+      context 'when rdf format error' do
+        before do
+          stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369').to_return(status: 200)
+          allow(RDF::Graph).to receive(:load).and_raise(RDF::FormatError)
+        end
+        it 'returns 500' do
+          msg = "RDF Format Error - Results from fetch term http://test.org/530369 for authority LOD_TERM_URI_PARAM_CONFIG was not identified as a valid RDF format.  " \
+                "You may need to include the linkeddata gem."
+          expect(Rails.logger).to receive(:warn).with(msg)
+          get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
+          expect(response.code).to eq('500')
+        end
+      end
+      context "when error isn't specifically handled" do
+        before do
+          stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369').to_return(status: 501)
+        end
+        it 'returns 500' do
+          expect(Rails.logger).to receive(:warn).with("Internal Server Error - Fetch term http://test.org/530369 unsuccessful for authority LOD_TERM_URI_PARAM_CONFIG")
+          get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
+          expect(response.code).to eq('500')
+        end
+      end
+    end
+    context 'when service unavailable' do
+      before do
+        stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369').to_return(status: 503)
+      end
+      it 'returns 503' do
+        expect(Rails.logger).to receive(:warn).with("Service Unavailable - Fetch term http://test.org/530369 unsuccessful for authority LOD_TERM_URI_PARAM_CONFIG")
+        get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
+        expect(response.code).to eq('503')
+      end
+    end
+    context 'when requested term is not found at the server' do
+      before do
+        stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/FAKE_ID').to_return(status: 404, body: '', headers: {})
+      end
+      it 'returns 404' do
+        expect(Rails.logger).to receive(:warn).with('Term Not Found - Fetch term http://test.org/FAKE_ID unsuccessful for authority LOD_TERM_URI_PARAM_CONFIG')
+        get :fetch, params: { uri: 'http://test.org/FAKE_ID', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
+        expect(response.code).to eq('404')
+      end
+    end
+    context 'in LOD_TERM_URI_PARAM_CONFIG authority' do
+      context 'term found' do
+        before do
+          stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369')
+            .to_return(status: 200, body: webmock_fixture('lod_oclc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
+        end
+        it 'succeeds and defaults to json content type' do
+          get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
+          expect(response).to be_successful
+          expect(response.content_type).to eq 'application/json'
+        end
+        context 'and it was requested as json' do
+          it 'succeeds and returns term data as json content type' do
+            get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG', format: 'json' }
+            expect(response).to be_successful
+            expect(response.content_type).to eq 'application/json'
+          end
+        end
+        context 'and it was requested as jsonld' do
+          it 'succeeds and returns term data as jsonld content type' do
+            get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG', format: 'jsonld' }
+            expect(response).to be_successful
+            expect(response.content_type).to eq 'application/ld+json'
+            expect(JSON.parse(response.body).keys).to match_array ["@context", "@graph"]
+          end
+        end
+      end
+      context 'when cors headers are enabled' do
+        before do
+          Qa.config.enable_cors_headers
+          stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369')
+            .to_return(status: 200, body: webmock_fixture('lod_oclc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
+        end
+        it 'Access-Control-Allow-Origin is *' do
+          get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
+          expect(response.headers['Access-Control-Allow-Origin']).to eq '*'
+        end
+      end
+      context 'when cors headers are disabled' do
+        before do
+          Qa.config.disable_cors_headers
+          stub_request(:get, 'http://localhost/test_default/term?uri=http://test.org/530369')
+            .to_return(status: 200, body: webmock_fixture('lod_oclc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
+        end
+        it 'Access-Control-Allow-Origin is not present' do
+          get :fetch, params: { uri: 'http://test.org/530369', vocab: 'LOD_TERM_URI_PARAM_CONFIG' }
+          expect(response.headers.key?('Access-Control-Allow-Origin')).to be false
+        end
+      end
+    end
+  end
+  describe '#reload' do
+    before do
+      Qa.config.authorized_reload_token = 'A_TOKEN'
+    end
+    context 'when token does not match' do
+      it 'returns 401' do
+        get :reload, params: { auth_token: 'BAD_TOKEN' }
+        expect(response.code).to eq('401')
+      end
+    end
+    context 'when token does match' do
+      it 'returns 200' do
+        get :reload, params: { auth_token: 'A_TOKEN' }
+        expect(response.code).to eq('200')
+      end
     end
   end
 end