bento_search 0.6.0 → 0.7.0

data/app/models/bento_search/results/pagination.rb

@@ -5,6 +5,9 @@
 #
 # You don't normally create one of these yourself, you get one returned
 # from Results#pagination
+#
+#
+#  <%= paginate @results.pagination %>
 class BentoSearch::Results::Pagination
   
   # first arg is results.total_items, second is result
@@ -12,9 +15,11 @@ class BentoSearch::Results::Pagination
   #
   # We don't do the page/start normalization calc here,
   # we count on them both being passed in, already calculated
-  # by normalize_arguments in SearchResults. Expect :page, 0-based
-  # :start, and :per_page
+  # by normalize_arguments in SearchResults. Expects :page, 0-based
+  # :start, and 1-based :per_page to all be passed in in initializer. 
   def initialize(total, normalized_args)
+    normalized_args ||= {} # in some error cases, we end up with nil
+    
     @total_count = total || 0
     @per_page = normalized_args[:per_page] || 10
     @current_page = normalized_args[:page]  || 1
@@ -42,6 +47,7 @@ class BentoSearch::Results::Pagination
   def count_records
     @total_count
   end
+  alias count count_records
   
   def total_pages
     (@total_count.to_f / @per_page).ceil

data/app/models/bento_search/search_engine.rb

@@ -49,13 +49,11 @@ module BentoSearch
   # Additional standard configuration keys that are implemented by the bento_search
   # framework:
   #
-  #  [item_decorators]
-  #      Array of Modules (or strings specifying modules, helpful to keep
-  #      config serializable) that will be decorated on to each individual search
-  #      BentoSearch::ResultItem. These can be used to, via configuration, change
-  #      the links associated with items, change certain item behaviors, or massage
-  #      item metadata. (Needs more documentation). 
-  #      
+  #  [for_display.decorator]
+  #      String name of decorator class that will be applied by #bento_decorate
+  #      helper in standard view. See wiki for more info on decorators. Must be
+  #      string name, actual class object not supported (to make it easier
+  #      to serialize and transport configuration). 
   #
   # == Implementing a SearchEngine
   #
@@ -143,8 +141,7 @@ module BentoSearch
       # merge in current instance config
       self.configuration.configure ( aConfiguration )
       
-      # global defaults?
-      self.configuration[:item_decorators] ||= []
+      # global defaults?      
       self.configuration[:for_display] ||= {}
             
       # check for required keys -- have to be present, and not nil
@@ -185,6 +182,9 @@ module BentoSearch
     #
     #  Ask an engine what semantic field names it supports with `engine.semantic_search_keys`
     #
+    #  Unrecognized search fields will be ignored, unless you pass in 
+    #  :unrecognized_search_field => :raise (or do same in config). 
+    #
     #  Ask an engine what sort fields it supports with `engine.sort_keys`. See
     #  list of standard sort keys in I18n file at ./config/locales/en.yml, in
     #  `en.bento_search.sort_keys`. 
@@ -207,8 +207,7 @@ module BentoSearch
       arguments = normalized_search_arguments(*arguments)
 
       results = search_implementation(arguments)
-      
-      decorate(results)      
+            
       
       # standard result metadata
       results.start = arguments[:start] || 0
@@ -220,6 +219,7 @@ module BentoSearch
       results.timing = (Time.now - start_t)
             
       results.display_configuration = configuration.for_display
+      results.each {|item| item.decorator = configuration.lookup!("for_display.decorator") }
         
       return results
     rescue *auto_rescue_exceptions => e
@@ -301,14 +301,17 @@ module BentoSearch
       
       # translate semantic_search_field to search_field, or raise if
       # can't. 
-      if (semantic = arguments.delete(:semantic_search_field)) && ! semantic.blank?
-        
+      if (semantic = arguments.delete(:semantic_search_field)) && ! semantic.blank?        
         mapped = self.semantic_search_map[semantic.to_s]
-        unless mapped
+        if config_arg(arguments, :unrecognized_search_field) == "raise" && ! mapped 
           raise ArgumentError.new("#{self.class.name} does not know about :semantic_search_field #{semantic}")
         end
         arguments[:search_field] = mapped
+      end      
+      if config_arg(arguments, :unrecognized_search_field) == "raise" && ! search_keys.include?(arguments[:search_field])
+        raise ArgumentError.new("#{self.class.name} does not know about :search_field #{arguments[:search_field]}")
       end
+        
               
       return arguments
     end
@@ -328,16 +331,19 @@ module BentoSearch
     
     protected
     
-    # Extend each result with each specified decorator module
-    # configuration.item_decorators is an array of either Module constants,
-    # or strings specifying module constants. 
-    def decorate(results)      
-      results.each do |result|
-        configuration.item_decorators.each do |decorator|
-          decorator = (decorator.kind_of? Module) ? decorator : BentoSearch::Util.constantize(decorator)          
-          result.extend decorator
-        end
+    # get value of an arg that can be supplied in search args OR config,
+    # with search_args over-ridding config. Also normalizes value to_s
+    # (for symbols/strings). 
+    def config_arg(arguments, key, default = nil)
+      value = if arguments[key].present?
+        arguments[key]
+      else
+        configuration[key]
       end
+      
+      value = value.to_s if value.kind_of? Symbol
+      
+      return value
     end
     
     # What exceptions should our #search wrapper rescue and turn

data/app/search_engines/bento_search/ebsco_host_engine.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require 'nokogiri'
 
 require 'http_client_patch/include_client'
@@ -10,7 +12,41 @@ require 'httpclient'
 #
 # * profile_id
 # * profile_password
-# * databases: ARRAY of ebsco shortcodes of what databases to include in search. If you specify one you don't have access to, you get an error message from ebsco, alas. 
+# * databases: ARRAY of ebsco shortcodes of what databases to include in search. If you specify one you don't have access to, you get an error message from ebsco, alas.
+#
+#
+# == Limits
+#
+# While waiting for a future bento_box generalized limit api, this engine
+# accepts custom search arguments to apply limits:
+#
+# [:peer_reviewed_only]   Set to boolean true or string 'true', to restrict
+#                         results to peer-reviewed only. (Or ask EBSCOHost
+#                         api to do so, what we get is what we get). 
+# [:pubyear_start]
+# [:pubyear_end]          Date range limiting, pass in custom search args,
+#                         one or both of pubyear_start and pubyear_end
+#                         #to_i will be called on it, so can be string. 
+#                         .search(:query => "foo", :pubyear_start => 2000)
+# 
+# == Custom response data
+# 
+# Iff EBSCO API reports that fulltext is available for the hit, then 
+# result.custom_data["fulltext_formats"] will be non-nil, and will be an array of
+# one or more of EBSCO's internal codes (P=PDF, T=HTML, C=HTML+Images). If
+# no fulltext is avail according to EBSCO API, result.custom_data["fulltext_formats"]
+# will be nil. 
+#
+# #link_is_fulltext also set to true/false
+#
+# You can use this to, for instance, hyperlink the displayed title directly
+# to record on EBSCO if and only if there's fulltext.  By writing a custom
+# decorator. See wiki on decorators. 
+#
+# == Limitations
+# We do set language of ResultItems based on what ebsco tells us, but ebsoc
+# seems to often leave out language or say 'english' for things that are not
+# (maybe cause abstract is in English?). Config variable to tell us to ignore language?
 #
 # == Note on including databases
 #
@@ -59,12 +95,11 @@ require 'httpclient'
 # * DTD of XML Response, hard to interpret but all we've got: http://support.ebsco.com/eit/docs/DTD_EIT_WS_searchResponse.zip
 #
 #  Hard to find docs page on embedding EBSCO limiters (like peer reviewed only "RV Y") in search query: 
-#     http://eit.ebscohost.com/Pages/MethodDescription.aspx?service=~/Services/SearchService.asmx&method=Info
+#     http://support.epnet.com/knowledge_base/detail.php?id=5397
 #
-# == Limitations
-# We do set language of ResultItems based on what ebsco tells us, but ebsoc
-# seems to tell us 'english' for everything (maybe cause abstract is in
-# English?). Config variable to tell us to ignore language?
+#  EBSCO searchable support portal has a section for the EIT api we use here:
+#     http://support.epnet.com/knowledge_base/search.php?keyword=&interface_id=1082&document_type=&page_function=search
+
 class BentoSearch::EbscoHostEngine
   include BentoSearch::SearchEngine
   
@@ -100,6 +135,7 @@ class BentoSearch::EbscoHostEngine
          (fault = xml.at_xpath("./Fault")))
     
          results.error ||= {}
+         results.error[:api_url] = url
          results.error[:exception] = exception if exception
          results.error[:status] = response.status if response
          
@@ -126,6 +162,19 @@ class BentoSearch::EbscoHostEngine
     
   end
   
+  # pass in nokogiri record xml for the records/rec node. 
+  # Returns nil if NO fulltext is avail on ebsco platform, 
+  # non-nil if fulltext is available. Non-nil value will
+  # actually be a non-empty ARRAY of internal EBSCO codes, P=PDF, T=HTML, C=HTML with images. 
+  # http://support.epnet.com/knowledge_base/detail.php?topic=996&id=3778&page=1
+  def fulltext_formats(record_xml)
+    fulltext_formats = record_xml.xpath("./header/controlInfo/artinfo/formats/fmt/@type").collect {|n| n.text }
+    
+    return nil if fulltext_formats.empty?
+    
+    return fulltext_formats    
+  end
+  
   
   # Pass in a nokogiri node, return node.text, or nil if
   # arg was nil or node.text was blank?
@@ -144,13 +193,22 @@ class BentoSearch::EbscoHostEngine
   def sniff_format(xml_node)
     return nil if xml_node.nil?
     
-
-    if xml_node.at_xpath("./jinfo/*") && xml_node.at_xpath("./artinfo/*")
+    if xml_node.at_xpath("./dissinfo/*")
+      :dissertation
+    elsif xml_node.at_xpath("./jinfo/*") && xml_node.at_xpath("./artinfo/*")
       "Article"
+    elsif xml_node.at_xpath("./dissinfo/disstl")
+      :dissertation
+    elsif xml_node.at_xpath("./bkinfo") && xml_node.at_xpath("./chapinfo")
+      :book_item
+    elsif xml_node.at_xpath("./bkinfo/btl") && xml_node.at_xpath("./artinfo/tig/atl") &&
+        (text_if_present(xml_node.at_xpath "./bkinfo/btl") != text_if_present(xml_node.at_xpath "./artinfo/tig/atl"))
+      # pathological case of book_item, if it has a bkinfo and an artinfo
+      # but the titles in both sections MATCH, it's just a book. If they're
+      # differnet, it's a book section, bah@
+      :book_item        
     elsif xml_node.at_xpath("./bkinfo/*")
       "Book"
-    elsif xml_node.at_xpath("./dissinfo/*")
-      :dissertation
     elsif xml_node.at_xpath("./jinfo/*")
       :serial
     else
@@ -174,10 +232,15 @@ class BentoSearch::EbscoHostEngine
     components = components.collect {|a| a.titlecase if a}
     components.uniq! # no need to have the same thing twice
 
+    
     # some hard-coded cases for better user-displayable string, and other
     # normalization. 
     if ["Academic Journal", "Journal"].include?(components.first) && ["Article", "Journal Article"].include?(components.last)
       return "Journal Article"
+    elsif components.last == "Book: Monograph"
+      return "Book" # Book: Monograph what??
+    elsif components.first == "Book Article"
+      return "Book Chapter"
     elsif components.first == "Periodical" && components.length > 1
       return components.last
     elsif components.size == 2 && components.first.include?(components.last)
@@ -207,7 +270,19 @@ class BentoSearch::EbscoHostEngine
     
     # undocumented but question mark seems to cause a problem for ebsco,
     # even inside quoted phrases, not sure why. 
-    txt.gsub(/[)(\?]/, ' ')
+    txt = txt.gsub(/[)(\?]/, ' ')
+    
+    # 'and' and 'or' need to be in phrase quotes to avoid being
+    # interpreted as boolean. For instance, when people just
+    # paste in a title: << A strategy for decreasing anxiety of ICU transfer patients and their families >>
+    # You'd think 'and' as boolean would still work there, but it resulted
+    # in zero hits unless quoted, I dunno. lowercase and uppercase and/or/not
+    # both cause observed weirdness. 
+    if ['and', 'or', 'not'].include?( txt.downcase )
+      txt = %Q{"#{txt}"}
+    end    
+    
+    return txt
   end
   
   # Actually turn the user's query into an EBSCO "AND" boolean query,
@@ -223,14 +298,12 @@ class BentoSearch::EbscoHostEngine
       ebsco_query_escape(t)      
     end
     
-    # Remove boolean operators if they are bare not in a phrase, they'll
-    # make things weird. In phrase quotes they are okay. 
+
     # Remove empty strings. Remove terms that are solely punctuation
     # without any letters. 
     terms.delete_if do |term|
       ( 
         term.blank? || 
-        ["AND", "OR", "NOT"].include?(term) ||
         term =~ /\A[^[[:alnum:]]]+\Z/
       )
     end
@@ -251,6 +324,22 @@ class BentoSearch::EbscoHostEngine
       query = "(#{args[:search_field]} #{query})"
     end
     
+    # peer-reviewed only?
+    if [true, "true"].include? args[:peer_reviewed_only]
+      query += " AND (RV Y)"
+    end
+    
+    if args[:pubyear_start] || args[:pubyear_end]
+      from = args[:pubyear_start].to_i 
+      from = nil if from == 0
+      
+      to = args[:pubyear_end].to_i 
+      to = nil if to == 0
+      
+      query += " AND (DT #{from}-#{to})"
+    end
+    
+    
     url += "&query=#{CGI.escape query}"
     
     # startrec is 1-based for ebsco, not 0-based like for us. 
@@ -280,8 +369,33 @@ class BentoSearch::EbscoHostEngine
     item.link           = get_link(xml_rec)
 
     item.issn           = text_if_present info.at_xpath("./jinfo/issn")
-
-    item.journal_title  = text_if_present(info.at_xpath("./jinfo/jtl"))
+    
+    # Dealing with titles is a bit crazy, while articles usually have atitles and
+    # jtitles, sometimes they have a btitle instead. A book will usually have
+    # both btitle and atitle, but sometimes just atitle. Book chapter, oh boy. 
+    
+    jtitle        = text_if_present(info.at_xpath("./jinfo/jtl"))
+    btitle        = text_if_present info.at_xpath("./bkinfo/btl")
+    atitle        = text_if_present info.at_xpath("./artinfo/tig/atl")
+    
+    if jtitle && atitle
+      item.title          = atitle
+      item.source_title   = jtitle
+    elsif btitle && atitle && atitle != btitle
+      # for a book, sometimes there's an atitle block and a btitle block
+      # when they're identical, this ain't a book section, it's a book. 
+      item.title          = atitle
+      item.source_title   = btitle
+    else
+      item.title  = atitle || btitle
+    end    
+    # EBSCO sometimes has crazy long titles, truncate em.
+    if item.title.present?
+      item.title        = text_helper.truncate(item.title, :length => 200, :separator => ' ', :omission => '…')
+    end
+    
+    
+    
     item.publisher      = text_if_present info.at_xpath("./pubinfo/pub")
     # if no publisher, but a dissertation institution, use that
     # as publisher. 
@@ -294,11 +408,22 @@ class BentoSearch::EbscoHostEngine
     item.isbn           = text_if_present info.at_xpath("./bkinfo/isbn")
     
     item.year           = text_if_present info.at_xpath("./pubinfo/dt/@year")
+    # fill in complete publication_date too only if we've got it. 
+    if (item.year &&
+        (month = text_if_present info.at_xpath("./pubinfo/dt/@month")) &&
+        (day = text_if_present info.at_xpath("./pubinfo/dt/@day"))      
+      )
+      if (item.year.to_i != 0 && month.to_i != 0 && day.to_i != 0)
+        item.publication_date = Date.new(item.year.to_i, month.to_i, day.to_i)        
+      end
+    end
+    
     item.volume         = text_if_present info.at_xpath("./pubinfo/vid")
     item.issue          = text_if_present info.at_xpath("./pubinfo/iid")
     
-    # EBSCO sometimes has crazy long titles, truncate em.     
-    item.title          = text_helper.truncate( text_if_present( info.at_xpath("./artinfo/tig/atl") ), :length => 200)
+    
+    
+    
     item.start_page     = text_if_present info.at_xpath("./artinfo/ppf")
     
     item.doi            = text_if_present info.at_xpath("./artinfo/ui[@type='doi']")
@@ -315,15 +440,21 @@ class BentoSearch::EbscoHostEngine
       item.authors << a
     end
    
-    
-    item.format         = sniff_format info
-    item.format_str     = sniff_format_str info
+    item.format          = sniff_format info
+    item.format_str      = sniff_format_str info
     
     # Totally unreliable, seems to report english for everything? Maybe
-    # because abstracts are in english? Nevertheless we include for now.
+    # because abstracts are in english? Nevertheless we include for now.    
     item.language_code   = text_if_present info.at_xpath("./language/@code")
-    
-    
+    # why does EBSCO return 'undetermined' sometimes? That might as well be
+    # not there, bah. 
+    item.language_code = nil if item.language_code == "und"
+    
+    # array of custom ebsco codes (or nil) for fulltext formats avail. 
+    item.custom_data["fulltext_formats"] = fulltext_formats xml_rec
+    # if any fulltext format, mark present
+    item.link_is_fulltext = item.custom_data["fulltext_formats"].present? 
+        
     return item
   end
   
@@ -343,6 +474,10 @@ class BentoSearch::EbscoHostEngine
     return noko
   end
   
+  def public_settable_search_args
+    super + [:peer_reviewed_only, :pubyear_start, :pubyear_end]
+  end
+  
   # David Walker says pretty much only relevance and date are realiable
   # in EBSCOhost cross-search. 
   def sort_definitions
@@ -351,9 +486,10 @@ class BentoSearch::EbscoHostEngine
       "date_desc" => {:implementation => "date"}
     }      
   end
-  
+    
   def search_field_definitions
     {
+      nil     => {:semantic => :general},
       "AU"    => {:semantic => :author},
       "TI"    => {:semantic => :title},
       "SU"    => {:semantic => :subject},

data/app/search_engines/bento_search/eds_engine.rb

@@ -21,11 +21,6 @@ require 'http_client_patch/include_client'
 # If turned on, you may get <b class="bento_search_highlight"> tags
 # in title and abstract output if it's on, marked html_safe.  
 #
-# If highlighting is on, since the abstract will be marked html safe, the
-# view layer won't be able to safely truncate it. In fact, it's very hard
-# to do here too, but we do it anyway, by default to approx configuration
-# truncate_highlighted num of chars (default 280). Set to nil if you don't
-# want this. 
 #
 # == Linking
 #
@@ -394,45 +389,7 @@ class BentoSearch::EdsEngine
       else substr.html_safe
       end
     end
-    
-    
-    
-    
-    
-    # Crazy ass method to truncate without getting in the middle of our
-    # html tags. This is wacky hacky, yeah. 
-    if configuration.truncate_highlighted
-      remainingLength = configuration.truncate_highlighted
-      in_tag = false
-      elipses_added = false
-      
-      truncated_parts = []
-      parts.each do |substr|
-        if remainingLength <=0 && ! in_tag
-          truncated_parts << "..."
-          break
-        end
         
-        if substr =~ /^<b.*\>$/
-          truncated_parts << substr
-          in_tag = true
-        elsif substr == "</b>"
-          truncated_parts << substr
-          in_tag = false
-        elsif ((remainingLength - substr.length) > 0) || in_tag
-          truncated_parts << substr
-        else
-          truncated_parts << helper.truncate(substr, :length => remainingLength, :separator  => ' ')
-          break
-        end
-        
-        remainingLength = remainingLength - substr.length
-      end
-      
-      parts = truncated_parts
-    end
-
-    
     return helper.safe_join(parts, '')        
   end
   
@@ -552,7 +509,7 @@ class BentoSearch::EdsEngine
   
   def search_field_definitions
     {
-      "TX" => {:semantic => :all},
+      "TX" => {:semantic => :general},
       "AU" => {:semantic => :author},
       "TI" => {:semantic => :title},
       "SU" => {:semantic => :subject},