ebsco-eds 0.3.14.pre → 0.3.15.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e2cd745f89b12f16e7db05e16bb5ad97d3545e68
-  data.tar.gz: 77c9f067970412617da0219fc60acf1e395d302e
+  metadata.gz: c0ed32f227645931d31d5ee595ca1b8793f4be7f
+  data.tar.gz: b5085f8c0c463f7bf5b084104119baf15befff4a
 SHA512:
-  metadata.gz: e62963cfd798ac3b22c24d331db5fa4512f928eb124ee0a7ca6dbcd1fb2029e550e913e6c86f625ee2380d06402855296733b348fa02c0d74bfd2666a2243d37
-  data.tar.gz: eb3e9b97eeb05a9d7a286b96b31d4e6ebf9caed16e23d0037d8425988824f5ee3007499e474026759fb94637a06c555013099a5838fdf60f04da23ab0fa0e410
+  metadata.gz: 72e23e4ee620db0170f53541684aa44d83f33822a44ec1d896f258c32908eeab0accf8041a860438e631e5dc0fe7b5366a48c205d68e8b293236c37abf1c8fcb
+  data.tar.gz: 929364e6faa8bebaa6f30288806cca3dff1d06bb3319c6603fab2091cf2ca7f76b36b9c9d2481fdf4dc8ff5ad2a489f4f62b2edcff5a0e598b1c465499237a3f

data/lib/ebsco/eds/configuration.rb

@@ -38,7 +38,9 @@ module EBSCO
             :open_timeout => 12,
             :max_page_jumps => 6,
             :max_page_jump_attempts => 10,
-            :recover_from_bad_source_type => false
+            :recover_from_bad_source_type => false,
+            :all_subjects_search_links => false,
+            :decode_sanitize_html => false
         }
         @valid_config_keys = @config.keys
       end

data/lib/ebsco/eds/options.rb

@@ -244,6 +244,8 @@ module EBSCO
                     _field_code = 'SH'
                   when 'keywords'
                     _field_code = 'KW'
+                  when /[A-Z]{2}/
+                    _field_code = _field
                 end
               end
 

data/lib/ebsco/eds/record.rb

@@ -121,7 +121,6 @@ module EBSCO
           'Related ISBNs'
       ]
 
-
       # Raw record as returned by the \EDS API via search or retrieve
       attr_accessor(*ATTRIBUTES)
 
@@ -130,7 +129,25 @@ module EBSCO
       end
 
       # Creates a search or retrieval result record
-      def initialize(results_record)
+      def initialize(results_record, eds_config = nil)
+
+        # translate all subject search link field codes to DE?
+        @all_subjects_search_links = false
+        if eds_config
+          @all_subjects_search_links = eds_config[:all_subjects_search_links]
+        end
+        if ENV.has_key? 'EDS_ALL_SUBJECTS_SEARCH_LINKS'
+          @all_subjects_search_links = ENV['EDS_ALL_SUBJECTS_SEARCH_LINKS']
+        end
+
+        # decode and sanitize html in item data?
+        @decode_sanitize_html = false
+        if eds_config
+          @decode_sanitize_html = eds_config[:decode_sanitize_html]
+        end
+        if ENV.has_key? 'EDS_DECODE_SANITIZE_HTML'
+          @decode_sanitize_html = ENV['EDS_DECODE_SANITIZE_HTML']
+        end
 
         if results_record.key? 'Record'
           @record = results_record['Record'] # single record returned by retrieve api
@@ -169,6 +186,7 @@ module EBSCO
         @eds_subjects =
             get_item_data({name: 'Subject', label: 'Subject Terms', group: 'Su'}) ||
             get_item_data({name: 'Subject', label: 'Subject Indexing', group: 'Su'}) ||
+            get_item_data({name: 'Subject', label: 'Subject Category', group: 'Su'}) ||
             bib_subjects
         @eds_subjects_geographic =
             get_item_data({name: 'SubjectGeographic', label: 'Geographic Terms', group: 'Su'}) ||
@@ -295,7 +313,40 @@ module EBSCO
       # Fulltext - RETRIEVE ONLY
       def html_fulltext
         if @record.fetch('FullText',{}).fetch('Text',{}).fetch('Availability',0) == '1'
-          @record.fetch('FullText',{}).fetch('Text',{})['Value']
+
+          # sanitize?
+          if @decode_sanitize_html
+
+            # transformer
+            clean_fulltext = lambda do |env|
+              node = env[:node]
+              if node.name == 'title'
+                node.name = 'h1'
+              end
+              if node.name == 'sbt'
+                node.name = 'h2'
+              end
+              if node.name == 'jsection'
+                node.name = 'h3'
+              end
+              if node.name == 'et'
+                node.name = 'h3'
+              end
+              node
+            end
+
+            fulltext_config = Sanitize::Config.merge(Sanitize::Config::RELAXED,
+                                                     :elements => Sanitize::Config::RELAXED[:elements] +
+                                                        %w[relatesto searchlink],
+                                                    :attributes => Sanitize::Config::RELAXED[:attributes].merge(
+                                                        'searchlink' => %w[fieldcode term]),
+            :remove_contents => true,
+            :transformers => [clean_fulltext])
+
+            html_decode_and_sanitize(@record.fetch('FullText',{}).fetch('Text',{})['Value'], fulltext_config)
+          else
+            @record.fetch('FullText',{}).fetch('Text',{})['Value']
+          end
         else
           nil
         end
@@ -757,8 +808,7 @@ module EBSCO
 
             @items.each do |item|
               if item['Name'] == options[:name] && item['Label'] == options[:label] && item['Group'] == options[:group]
-                # puts 'FOUND ALL 3: ' + item.inspect
-                return sanitize_data(item['Data'])
+                return sanitize_data(item)
               end
             end
             return nil
@@ -767,7 +817,7 @@ module EBSCO
 
             @items.each do |item|
               if item['Name'] == options[:name] && item['Label'] == options[:label]
-                return sanitize_data(item['Data'])
+                return sanitize_data(item)
               end
             end
             return nil
@@ -776,7 +826,7 @@ module EBSCO
 
             @items.each do |item|
               if item['Name'] == options[:name] && item['Group'] == options[:group]
-                return sanitize_data(item['Data'])
+                return sanitize_data(item)
               end
             end
             return nil
@@ -785,7 +835,7 @@ module EBSCO
 
             @items.each do |item|
               if item['Label'] == options[:label] && item['Group'] == options[:group]
-                return sanitize_data(item['Data'])
+                return sanitize_data(item)
               end
             end
             return nil
@@ -794,7 +844,7 @@ module EBSCO
 
             @items.each do |item|
               if item['Label'] == options[:label]
-                return sanitize_data(item['Data'])
+                return sanitize_data(item)
               end
             end
             return nil
@@ -803,7 +853,7 @@ module EBSCO
 
             @items.each do |item|
               if item['Name'] == options[:name]
-                return sanitize_data(item['Data'])
+                return sanitize_data(item)
               end
             end
             return nil
@@ -815,16 +865,47 @@ module EBSCO
         end
       end
 
-      # sanitize html, allow custom links
-      def sanitize_data(data)
-        html = CGI.unescapeHTML(data.to_s)
-        sanitize_config = Sanitize::Config.merge(Sanitize::Config::RELAXED,
-                                                 :elements => Sanitize::Config::RELAXED[:elements] + ['relatesto', 'searchlink'],
+      # decode & sanitize html tags found in item data; apply any special transformations
+      def sanitize_data(item)
+
+        if item['Data']
+          data = item['Data']
+
+          # group-specific transformations
+          if item['Group']
+            group = item['Group']
+            if group == 'Su'
+              # translate searchLink field codes to DE?
+              if @all_subjects_search_links
+                data = data.gsub(/(searchLink fieldCode=")([A-Z]+)/, '\1DE')
+              end
+            end
+          end
+
+          # decode-sanitize?
+          if @decode_sanitize_html
+           data = html_decode_and_sanitize(data)
+          end
+
+          data
+
+        else
+          nil # no item data present
+        end
+
+      end
+
+      # Decode any html elements and then run it through sanitize to preserve entities (eg: ampersand) and strip out
+      # elements/attributes that aren't explicitly whitelisted.
+      # The RELAXED config: https://github.com/rgrove/sanitize/blob/master/lib/sanitize/config/relaxed.rb
+      def html_decode_and_sanitize(data, config = nil)
+        default_config = Sanitize::Config.merge(Sanitize::Config::RELAXED,
+                                                 :elements => Sanitize::Config::RELAXED[:elements] +
+                                                     %w[relatesto searchlink],
                                                  :attributes => Sanitize::Config::RELAXED[:attributes].merge(
-                                                     'searchlink' => ['fieldcode', 'term']
-                                                 )
-        )
-        Sanitize.fragment(html, sanitize_config)
+                                                     'searchlink' => %w[fieldcode term]))
+        sanitize_config = config.nil? ? default_config : config
+        Sanitize.fragment(CGI.unescapeHTML(data.to_s), sanitize_config)
       end
 
       # dynamically add item metadata as 'eds_extra_ItemNameOrLabel'

data/lib/ebsco/eds/results.rb

@@ -28,7 +28,7 @@ module EBSCO
 
       # Creates search results from the \EDS API search response. It includes information about the results and a list
       # of Record items.
-      def initialize(search_results, additional_limiters = {}, options = {})
+      def initialize(search_results, eds_config = nil, additional_limiters = {}, options = {})
 
         @results = search_results
         @limiters = additional_limiters
@@ -39,7 +39,7 @@ module EBSCO
         if @results['SearchResult']['Data']['Records']
           @results['SearchResult']['Data']['Records'].each { |record|
 
-            @records.push(EBSCO::EDS::Record.new(record))
+            @records.push(EBSCO::EDS::Record.new(record, eds_config))
 
             # # records hidden in guest mode
             # if record['Header']['AccessLevel']
@@ -64,7 +64,7 @@ module EBSCO
               rs_entries = related_item.fetch('Records',{})
               if rs_entries.count > 0
                 rs_entries.each do |rs_record|
-                  @research_starters.push(EBSCO::EDS::Record.new(rs_record))
+                  @research_starters.push(EBSCO::EDS::Record.new(rs_record, eds_config))
                 end
               end
             end
@@ -80,7 +80,7 @@ module EBSCO
               _publication_matches = related_item.fetch('PublicationRecords',{})
               if _publication_matches.count > 0
                 _publication_matches.each do |publication_record|
-                  @publication_match.push(EBSCO::EDS::Record.new(publication_record))
+                  @publication_match.push(EBSCO::EDS::Record.new(publication_record, eds_config))
                 end
               end
             end

data/lib/ebsco/eds/session.rb

@@ -207,10 +207,11 @@ module EBSCO
         # use existing/updated SearchOptions
         if options.empty?
           if @search_options.nil?
-            @search_results = EBSCO::EDS::Results.new(empty_results)
+            @search_results = EBSCO::EDS::Results.new(empty_results,@config)
           else
             _response = do_request(:post, path: '/edsapi/rest/Search', payload: @search_options)
-            @search_results = EBSCO::EDS::Results.new(_response, @info.available_limiters, options)
+            @search_results = EBSCO::EDS::Results.new(_response, @config,
+                                                      @info.available_limiters, options)
             if increment_page
               @current_page = @search_results.page_number
             end
@@ -226,7 +227,8 @@ module EBSCO
             end
 
             _response = do_request(:post, path: '/edsapi/rest/Search', payload: @search_options)
-            @search_results = EBSCO::EDS::Results.new(_response, @info.available_limiters, options)
+            @search_results = EBSCO::EDS::Results.new(_response, @config,
+                                                      @info.available_limiters, options)
 
             # create temp format facet results if needed
             if options['f']
@@ -236,7 +238,10 @@ module EBSCO
                 format_search_options = EBSCO::EDS::Options.new(format_options, @info)
                 format_search_options.Comment = 'temp source type facets'
                 _format_response = do_request(:post, path: '/edsapi/rest/Search', payload: format_search_options)
-                @search_results.temp_format_facet_results = EBSCO::EDS::Results.new(_format_response, @info.available_limiters, format_options)
+                @search_results.temp_format_facet_results = EBSCO::EDS::Results.new(_format_response,
+                                                                                    @config,
+                                                                                    @info.available_limiters,
+                                                                                    format_options)
               end
             end
 
@@ -248,7 +253,10 @@ module EBSCO
                 content_search_options = EBSCO::EDS::Options.new(content_options, @info)
                 content_search_options.Comment = 'temp content provider facet'
                 _content_response = do_request(:post, path: '/edsapi/rest/Search', payload: content_search_options)
-                @search_results.temp_content_provider_facet_results = EBSCO::EDS::Results.new(_content_response, @info.available_limiters, content_options)
+                @search_results.temp_content_provider_facet_results = EBSCO::EDS::Results.new(_content_response,
+                                                                                              @config,
+                                                                                              @info.available_limiters,
+                                                                                              content_options)
               end
             end
 
@@ -297,8 +305,8 @@ module EBSCO
         retrieve_response = do_request(:post, path: @config[:retrieve_url], payload: payload)
         #retrieve_params = "?an=#{an}&dbid=#{dbid}&ebookpreferredformat=#{ebook}"
         #retrieve_response = do_request(:get, path: @config[:retrieve_url] + retrieve_params)
-        record = EBSCO::EDS::Record.new(retrieve_response)
-        # puts 'RECORD: ' + record.pretty_inspect
+        record = EBSCO::EDS::Record.new(retrieve_response, @config)
+        # puts 'RECORD: ' + record.inspect
         record
       end
 
@@ -357,7 +365,7 @@ module EBSCO
 
         # return json result set with just the previous and next records in it
         r = empty_results(cached_results.stat_total_hits)
-        results = EBSCO::EDS::Results.new(r)
+        results = EBSCO::EDS::Results.new(r, @config)
         next_previous_records = []
         unless result_prev.nil?
           next_previous_records << result_prev
@@ -380,7 +388,7 @@ module EBSCO
           }
         end
         r = empty_results(records.length)
-        results = EBSCO::EDS::Results.new(r)
+        results = EBSCO::EDS::Results.new(r, @config)
         results.records = records
         results.to_solr
       end
@@ -818,7 +826,7 @@ module EBSCO
                         else
                           if !action.include?('SourceType:'+bad_source_type+')')
                             # not a bad source type, keep it
-                            new_actions >> action
+                            new_actions << action
                           end
                         end
                       else
@@ -826,6 +834,31 @@ module EBSCO
                         new_actions << action
                       end
                     }
+
+                    new_filters = []
+                    filter_id = 1
+                    payload.SearchCriteria.FacetFilters.each { |filter|
+                      filter['FacetValues'].each { |facet_val|
+                        if facet_val['Id'] == 'SourceType'
+                          if bad_source_type.nil?
+                            # skip the source type since we don't know if it's bad or not
+                          else
+                            # not a bad sourcetype, add it
+                            if !facet_val['Value'].include?(bad_source_type)
+                            filter['FilterId'] = filter_id
+                            filter_id += 1
+                            new_filters << filter
+                            end
+                          end
+                        else
+                          # not a SourceType filter, add it
+                          filter['FilterId'] = filter_id
+                          filter_id += 1
+                          new_filters << filter
+                        end
+                      }
+                    }
+                    payload.SearchCriteria.FacetFilters = new_filters
                     payload.Actions = new_actions
                     do_request(method, path: path, payload: payload, attempt: attempt+1)
                   else

data/lib/ebsco/eds/version.rb

@@ -1,5 +1,5 @@
 module EBSCO
   module EDS
-    VERSION = '0.3.14.pre'
+    VERSION = '0.3.15.pre'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ebsco-eds
 version: !ruby/object:Gem::Version
-  version: 0.3.14.pre
+  version: 0.3.15.pre
 platform: ruby
 authors:
 - Bill McKinney
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-09-14 00:00:00.000000000 Z
+date: 2017-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday