qa 1.0.0 → 1.1.0

data/lib/qa/authorities/linked_data/search_query.rb

@@ -0,0 +1,143 @@
+# This module has the primary QA search method.  It also includes methods to process the linked data results and convert
+# them into the expected QA json results format.
+module Qa::Authorities
+  module LinkedData
+    class SearchQuery
+      include Qa::Authorities::LinkedData::RdfHelper
+
+      # @param [SearchConfig] search_config The search portion of the config
+      def initialize(search_config)
+        @search_config = search_config
+      end
+
+      attr_reader :search_config
+
+      delegate :subauthority?, :supports_sort?, to: :search_config
+
+      # Search a linked data authority
+      # @param [String] the query
+      # @param [Symbol] (optional) language: language used to select literals when multi-language is supported (e.g. :en, :fr, etc.)
+      # @param [Hash] (optional) replacements: replacement values with { pattern_name (defined in YAML config) => value }
+      # @param [String] subauth: the subauthority to query
+      # @return [String] json results
+      # @example Json Results for Linked Data Search
+      #   [ {"uri":"http://id.worldcat.org/fast/5140","id":"5140","label":"Cornell, Joseph"},
+      #     {"uri":"http://id.worldcat.org/fast/72456","id":"72456","label":"Cornell, Sarah Maria, 1802-1832"},
+      #     {"uri":"http://id.worldcat.org/fast/409667","id":"409667","label":"Cornell, Ezra, 1807-1874"} ]
+      def search(query, language: nil, replacements: {}, subauth: nil)
+        raise Qa::InvalidLinkedDataAuthority, "Unable to initialize linked data search sub-authority #{subauth}" unless subauth.nil? || subauthority?(subauth)
+        language ||= search_config.language
+        url = search_config.url_with_replacements(query, subauth, replacements)
+        Rails.logger.info "QA Linked Data search url: #{url}"
+        graph = get_linked_data(url)
+        parse_search_authority_response(graph, language)
+      end
+
+      private
+
+        def parse_search_authority_response(graph, language)
+          graph = filter_language(graph, language) unless language.nil?
+          results = extract_preds(graph, preds_for_search)
+          consolidated_results = consolidate_search_results(results)
+          json_results = convert_search_to_json(consolidated_results)
+          sort_search_results(json_results)
+        end
+
+        def preds_for_search
+          { required: required_search_preds, optional: optional_search_preds }
+        end
+
+        def required_search_preds
+          label_pred_uri = search_config.results_label_predicate
+          raise Qa::InvalidConfiguration, "required label_predicate is missing in search configuration for LOD authority #{auth_name}" if label_pred_uri.nil?
+          { label: label_pred_uri }
+        end
+
+        def optional_search_preds
+          preds = {}
+          preds[:altlabel] = search_config.results_altlabel_predicate unless search_config.results_altlabel_predicate.nil?
+          preds[:id] = search_config.results_id_predicate unless search_config.results_id_predicate.nil?
+          preds[:sort] = search_config.results_sort_predicate unless search_config.results_sort_predicate.nil?
+          preds
+        end
+
+        def consolidate_search_results(results)
+          consolidated_results = {}
+          return consolidated_results if results.nil? || !results.count.positive?
+          results.each do |statement|
+            stmt_hash = statement.to_h
+            uri = stmt_hash[:uri].to_s
+            consolidated_hash = init_consolidated_hash(consolidated_results, uri, stmt_hash[:id].to_s)
+
+            consolidated_hash[:label] = object_value(stmt_hash, consolidated_hash, :label, false)
+            consolidated_hash[:altlabel] = object_value(stmt_hash, consolidated_hash, :altlabel, false)
+            consolidated_hash[:sort] = object_value(stmt_hash, consolidated_hash, :sort, false)
+            consolidated_results[uri] = consolidated_hash
+          end
+          consolidated_results.each do |res|
+            consolidated_hash = res[1]
+            consolidated_hash[:label] = sort_string_by_language consolidated_hash[:label]
+            consolidated_hash[:altlabel] = sort_string_by_language consolidated_hash[:altlabel]
+            consolidated_hash[:sort] = sort_string_by_language consolidated_hash[:sort]
+          end
+          consolidated_results
+        end
+
+        def convert_search_to_json(consolidated_results)
+          json_results = []
+          consolidated_results.each do |uri, h|
+            json_results << { uri: uri, id: h[:id], label: full_label(h[:label], h[:altlabel]), sort: h[:sort] }
+          end
+          json_results
+        end
+
+        def full_label(label = [], altlabel = [])
+          lbl = wrap_labels(label)
+          lbl += " (#{altlabel.join(', ')})" unless altlabel.nil? || altlabel.length <= 0
+          lbl = lbl.slice(0..95) + '...' if lbl.length > 98
+          lbl.strip
+        end
+
+        def wrap_labels(labels)
+          lbl = "" if labels.nil? || labels.size.zero?
+          lbl = labels.join(', ') if labels.size.positive?
+          lbl = '[' + lbl + ']' if labels.size > 1
+          lbl
+        end
+
+        def sort_search_results(json_results)
+          return json_results unless supports_sort?
+          json_results.sort! do |a, b|
+            cmp = sort_when_missing_sort_predicate(a, b)
+            next unless cmp.nil?
+
+            as = a[:sort].collect(&:downcase)
+            bs = b[:sort].collect(&:downcase)
+            cmp = 0
+            0.upto([as.size, bs.size].max - 1) do |i|
+              cmp = sort_when_same_but_one_has_more_values(as, bs, i)
+              break unless cmp.nil?
+
+              cmp = (as[i] <=> bs[i])
+              break if cmp.nonzero? # stop checking as soon as a value in the two lists are different
+            end
+            cmp
+          end
+          json_results.each { |h| h.delete(:sort) }
+        end
+
+        def sort_when_missing_sort_predicate(a, b)
+          return 0 unless a.key?(:sort) || b.key?(:sort) # leave unchanged if both are missing
+          return -1 unless a.key? :sort # consider missing a value lower than existing b value
+          return 1 unless b.key? :sort # consider missing b value lower than existing a value
+          nil
+        end
+
+        def sort_when_same_but_one_has_more_values(as, bs, current_list_size)
+          return -1 if as.size <= current_list_size # consider shorter a list of values lower then longer b list
+          return 1 if bs.size <= current_list_size # consider shorter b list of values lower then longer a list
+          nil
+        end
+    end
+  end
+end

data/lib/qa/version.rb

@@ -1,3 +1,3 @@
 module Qa
-  VERSION = "1.0.0".freeze
+  VERSION = "1.1.0".freeze
 end

data/spec/controllers/linked_data_terms_controller_spec.rb

@@ -0,0 +1,202 @@
+require 'spec_helper'
+
+describe Qa::LinkedDataTermsController, type: :controller do
+  before do
+    @routes = Qa::Engine.routes
+  end
+
+  describe '#check_authority' do
+    it 'returns 404 if the vocabulary is not specified' do
+      expect(Rails.logger).to receive(:warn).with("Required param 'vocab' is missing or empty")
+      get :search, params: { q: 'a query', vocab: '' }
+      expect(response.code).to eq('404')
+    end
+
+    it 'returns 404 if the vocabulary is not specified' do
+      expect(Rails.logger).to receive(:warn).with("Required param 'vocab' is missing or empty")
+      get :show, params: { id: 'C_1234', vocab: '' }
+      expect(response.code).to eq('404')
+    end
+  end
+
+  describe '#check_search_subauthority' do
+    it 'returns 404 if the query subauthority is missing' do
+      expect(Rails.logger).to receive(:warn).with("Unable to initialize linked data search sub-authority '' for authority 'OCLC_FAST'")
+      get :search, params: { q: 'test', vocab: 'OCLC_FAST', subauthority: '' }
+      expect(response.code).to eq('404')
+    end
+    it 'returns 404 if the query subauthority is invalid' do
+      expect(Rails.logger).to receive(:warn).with("Unable to initialize linked data search sub-authority 'FAKE_SUBAUTHORITY' for authority 'OCLC_FAST'")
+      get :search, params: { vocab: 'OCLC_FAST', subauthority: 'FAKE_SUBAUTHORITY' }
+      expect(response.code).to eq('404')
+    end
+  end
+
+  describe '#check_show_subauthority' do
+    it 'returns 404 if the show subauthority is missing' do
+      expect(Rails.logger).to receive(:warn).with("Unable to initialize linked data term sub-authority '' for authority 'OCLC_FAST'")
+      get :show, params: { id: 'C_123', vocab: 'OCLC_FAST', subauthority: '' }
+      expect(response.code).to eq('404')
+    end
+    it 'returns 404 if the show subauthority is invalid' do
+      expect(Rails.logger).to receive(:warn).with("Unable to initialize linked data term sub-authority 'FAKE_SUBAUTHORITY' for authority 'OCLC_FAST'")
+      get :show, params: { id: 'C_123', vocab: 'OCLC_FAST', subauthority: 'FAKE_SUBAUTHORITY' }
+      expect(response.code).to eq('404')
+    end
+  end
+
+  describe '#check_query_param' do
+    it 'returns 404 if the query is missing' do
+      expect(Rails.logger).to receive(:warn).with("Required search param 'q' is missing or empty")
+      get :search, params: { vocab: 'OCLC_FAST' }
+      expect(response.code).to eq('404')
+    end
+  end
+
+  describe '#check_id_param' do
+    it 'returns 404 if the id is missing' do
+      expect(Rails.logger).to receive(:warn).with("Required show param 'id' is missing or empty")
+      get :show, params: { id: '', vocab: 'OCLC_FAST' }
+      expect(response.code).to eq('404')
+    end
+  end
+
+  describe '#init_authority' do
+    context 'when the authority does not exist' do
+      it 'returns 404' do
+        expect(Rails.logger).to receive(:warn).with("Unable to initialize linked data authority 'FAKE_AUTHORITY'")
+        get :search, params: { q: 'a query', vocab: 'fake_authority' }
+        expect(response.code).to eq('404')
+      end
+    end
+  end
+
+  describe '#search' do
+    context 'in OCLC_FAST authority' do
+      context '0 search results' do
+        before do
+          stub_request(:get, 'http://experimental.worldcat.org/fast/search?maximumRecords=3&query=cql.any%20all%20%22supercalifragilisticexpialidocious%22&sortKeys=usage')
+            .to_return(status: 200, body: webmock_fixture('lod_oclc_query_no_results.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
+        end
+        it 'succeeds' do
+          get :search, params: { q: 'supercalifragilisticexpialidocious', vocab: 'OCLC_FAST', maximumRecords: '3' }
+          expect(response).to be_success
+        end
+      end
+
+      context '3 search results' do
+        before do
+          stub_request(:get, 'http://experimental.worldcat.org/fast/search?maximumRecords=3&query=cql.any%20all%20%22cornell%22&sortKeys=usage')
+            .to_return(status: 200, body: webmock_fixture('lod_oclc_all_query_3_results.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
+        end
+        it 'succeeds' do
+          get :search, params: { q: 'cornell', vocab: 'OCLC_FAST', maximumRecords: '3' }
+          expect(response).to be_success
+        end
+      end
+    end
+
+    context 'in OCLC_FAST authority and personal_name subauthority' do
+      context '0 search results' do
+        before do
+          stub_request(:get, 'http://experimental.worldcat.org/fast/search?maximumRecords=3&query=oclc.personalName%20all%20%22supercalifragilisticexpialidocious%22&sortKeys=usage')
+            .to_return(status: 200, body: webmock_fixture('lod_oclc_query_no_results.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
+        end
+        it 'succeeds' do
+          get :search, params: { q: 'supercalifragilisticexpialidocious', vocab: 'OCLC_FAST', subauthority: 'personal_name', maximumRecords: '3' }
+          expect(response).to be_success
+        end
+      end
+
+      context '3 search results' do
+        before do
+          stub_request(:get, 'http://experimental.worldcat.org/fast/search?maximumRecords=3&query=oclc.personalName%20all%20%22cornell%22&sortKeys=usage')
+            .to_return(status: 200, body: webmock_fixture('lod_oclc_personalName_query_3_results.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
+        end
+        it 'succeeds' do
+          get :search, params: { q: 'cornell', vocab: 'OCLC_FAST', subauthority: 'personal_name', maximumRecords: '3' }
+          expect(response).to be_success
+        end
+      end
+    end
+
+    context 'in AGROVOC authority' do
+      context '0 search results' do
+        before do
+          stub_request(:get, 'http://aims.fao.org/skosmos/rest/v1/search/?lang=en&query=*supercalifragilisticexpialidocious*')
+            .to_return(status: 200, body: webmock_fixture('lod_agrovoc_query_no_results.json'), headers: { 'Content-Type' => 'application/json' })
+        end
+        it 'succeeds' do
+          get :search, params: { q: 'supercalifragilisticexpialidocious', vocab: 'AGROVOC' }
+          expect(response).to be_success
+        end
+      end
+
+      context '3 search results' do
+        before do
+          stub_request(:get, 'http://aims.fao.org/skosmos/rest/v1/search/?lang=en&query=*milk*')
+            .to_return(status: 200, body: webmock_fixture('lod_agrovoc_query_many_results.json'), headers: { 'Content-Type' => 'application/json' })
+        end
+        it 'succeeds' do
+          get :search, params: { q: 'milk', vocab: 'AGROVOC' }
+          expect(response).to be_success
+        end
+      end
+    end
+  end
+
+  describe '#show' do
+    context 'basic parameter testing' do
+      context 'with bad id' do
+        before do
+          stub_request(:get, 'http://id.worldcat.org/fast/FAKE_ID')
+            .to_return(status: 404, body: '', headers:  {})
+        end
+        it 'returns 404' do
+          expect(Rails.logger).to receive(:warn).with('Term Not Found - Fetch term FAKE_ID unsuccessful for authority OCLC_FAST')
+          get :show, params: { id: 'FAKE_ID', vocab: 'OCLC_FAST' }
+          expect(response.code).to eq('404')
+        end
+      end
+    end
+
+    context 'in OCLC_FAST authority' do
+      context 'term found' do
+        before do
+          stub_request(:get, 'http://id.worldcat.org/fast/530369')
+            .to_return(status: 200, body: webmock_fixture('lod_oclc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
+        end
+        it 'succeeds' do
+          get :show, params: { id: '530369', vocab: 'OCLC_FAST' }
+          expect(response).to be_success
+        end
+      end
+    end
+
+    context 'in AGROVOC authority' do
+      context 'term found' do
+        before do
+          stub_request(:get, 'http://aims.fao.org/skosmos/rest/v1/data?uri=http://aims.fao.org/aos/agrovoc/c_9513')
+            .to_return(status: 200, body: webmock_fixture('lod_agrovoc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
+        end
+        it 'succeeds' do
+          get :show, params: { id: 'c_9513', vocab: 'AGROVOC' }
+          expect(response).to be_success
+        end
+      end
+    end
+
+    context 'in LOC authority' do
+      context 'term found' do
+        before do
+          stub_request(:get, 'http://id.loc.gov/authorities/subjects/sh85118553')
+            .to_return(status: 200, body: webmock_fixture('lod_loc_term_found.rdf.xml'), headers: { 'Content-Type' => 'application/rdf+xml' })
+        end
+        it 'succeeds' do
+          get :show, params: { id: 'sh85118553', vocab: 'LOC', subauthority: 'subjects' }
+          expect(response).to be_success
+        end
+      end
+    end
+  end
+end

data/spec/fixtures/authorities/linked_data/lod_full_config.json

@@ -0,0 +1,111 @@
+{
+  "term": {
+    "url": {
+      "@context": "http://www.w3.org/ns/hydra/context.jsonld",
+      "@type":    "IriTemplate",
+      "template": "http://localhost/test_default/term/{?subauth}/{?term_id}?param1={?param1}&param2={?param2}",
+      "variableRepresentation": "BasicRepresentation",
+      "mapping": [
+        {
+          "@type":    "IriTemplateMapping",
+          "variable": "term_id",
+          "property": "hydra:freetextQuery",
+          "required": true
+        },
+        {
+          "@type":    "IriTemplateMapping",
+          "variable": "subauth",
+          "property": "hydra:freetextQuery",
+          "required": false,
+          "default": "term_sub2_name"
+        },
+        {
+          "@type": "IriTemplateMapping",
+          "variable": "param1",
+          "property": "hydra:freetextQuery",
+          "required": false,
+          "default": "alpha"
+        },
+        {
+          "@type": "IriTemplateMapping",
+          "variable": "param2",
+          "property": "hydra:freetextQuery",
+          "required": false,
+          "default": "beta"
+        }
+      ]
+    },
+    "qa_replacement_patterns": {
+      "term_id": "term_id",
+      "subauth": "subauth"
+    },
+    "term_id": "ID",
+    "language": [ "en" ],
+    "results": {
+      "id_predicate":       "http://purl.org/dc/terms/identifier",
+      "label_predicate":    "http://www.w3.org/2004/02/skos/core#prefLabel",
+      "altlabel_predicate": "http://www.w3.org/2004/02/skos/core#altLabel",
+      "broader_predicate":  "http://www.w3.org/2004/02/skos/core#broader",
+      "narrower_predicate": "http://www.w3.org/2004/02/skos/core#narrower",
+      "sameas_predicate":   "http://www.w3.org/2004/02/skos/core#exactMatch"
+    },
+    "subauthorities": {
+      "term_sub1_key": "term_sub1_name",
+      "term_sub2_key": "term_sub2_name",
+      "term_sub3_key": "term_sub3_name"
+    }
+  },
+  "search": {
+    "url": {
+      "@context": "http://www.w3.org/ns/hydra/context.jsonld",
+      "@type": "IriTemplate",
+      "template": "http://localhost/test_default/search?subauth={?subauth}&query={?query}&param1={?param1}&param2={?param2}",
+      "variableRepresentation": "BasicRepresentation",
+      "mapping": [
+        {
+          "@type": "IriTemplateMapping",
+          "variable": "query",
+          "property": "hydra:freetextQuery",
+          "required": true
+        },
+        {
+          "@type": "IriTemplateMapping",
+          "variable": "subauth",
+          "property": "hydra:freetextQuery",
+          "required": false,
+          "default": "search_sub1_name"
+        },
+        {
+          "@type": "IriTemplateMapping",
+          "variable": "param1",
+          "property": "hydra:freetextQuery",
+          "required": false,
+          "default": "delta"
+        },
+        {
+          "@type": "IriTemplateMapping",
+          "variable": "param2",
+          "property": "hydra:freetextQuery",
+          "required": false,
+          "default": "echo"
+        }
+      ]
+    },
+    "qa_replacement_patterns": {
+      "query": "query",
+      "subauth": "subauth"
+    },
+    "language": [ "en", "fr", "de" ],
+    "results": {
+      "id_predicate":       "http://purl.org/dc/terms/identifier",
+      "label_predicate":    "http://www.w3.org/2004/02/skos/core#prefLabel",
+      "altlabel_predicate": "http://www.w3.org/2004/02/skos/core#altLabel",
+      "sort_predicate":     "http://www.w3.org/2004/02/skos/core#prefLabel"
+    },
+    "subauthorities": {
+      "search_sub1_key": "search_sub1_name",
+      "search_sub2_key": "search_sub2_name",
+      "search_sub3_key": "search_sub3_name"
+    }
+  }
+}