bento_search 0.0.1 → 0.5.0

data/README.md

@@ -2,9 +2,9 @@
 
 [![Build Status](https://secure.travis-ci.org/jrochkind/bento_search.png)](http://travis-ci.org/jrochkind/bento_search)
 
-
-(**in progress*, not yet ready for use, mainly because we need more
-out of the box search engines supported). 
+(Fairly robust and stable at this point, but still pre-1.0 release, may
+be some breaking api changes before 1.0, but probably not too many, it's
+looking pretty good). 
 
 bento_search provides an abstraction/normalization layer for querying and 
 displaying results for external search engines, in Ruby on Rails. Requires
@@ -37,8 +37,13 @@ code in your app that needs to be rewritten. As well as letting you get
 started quick without reinventing the wheel and figuring out poorly
 documented vendor API's yourself. 
 
+See code-level api documentation for more details, especially at
+BentoSearch::SearchEngine. http://rubydoc.info/gems/bento_search/frames/
+
+An example app using BentoSearch and showing it's features is
+available at http://github.com/jrochkind/sample_megasearch
 
-## Usage
+## Usage Examples
 
 ### Instantiate an engine, and search
 
@@ -245,8 +250,8 @@ be used to add them.
        conf.item_decorators = [ SomeModule, OtherModule]
     end
 
-See BentoSearch::Item for more information on decorators, and BentoSearch::Link
-on links. 
+See BentoSearch::Link for more info on links. (TODO: Better docs/examples
+on decorators). 
 
 ## Planned Features
 
@@ -289,8 +294,9 @@ To re-generate cached responses, delete the relevant files in
 variable with your own API keys to re-run tests without cached response
 like this. 
 
-Also note `./test/support/mock_engine.rb`, a simple mock/dummy SearchEngine
-implementation that can be used in other tests. 
+Also note `BentoSearch::MockEngine`, a simple mock/dummy SearchEngine
+implementation that can be used in other tests, including in client
+software where convenient. 
 
 Pull requests welcome.  Pull requests with additional search engine implementations
 welcome. See more info on writing a BentoSearch::SearchEngine in the inline

data/app/assets/stylesheets/bento_search/suggested_styles.css

@@ -0,0 +1,22 @@
+/* some suggested styles for bento search results. You can
+   include this in your asset pipeline with `require bento_search/suggested_styles`,
+   or just use this as documentation for suggestions to implement yourself. */
+   
+   
+/* year in bold in citations */
+.bento_item_row.published_in .year {
+  font-weight: bold;
+}
+
+/* highlighted element in title is already in a <b> tag, but
+   title is likely to be bold already. italisize it too. */   
+.bento_item_title b.bento_search_highlight {
+  font-style: italic;
+}
+
+/* center the ajax_wait spinner */
+.bento_search_ajax_wait {
+  text-align: center;
+  margin: 2em;
+}
+

data/app/models/bento_search/multi_searcher.rb

@@ -92,7 +92,7 @@ class BentoSearch::MultiSearcher
       begin
         @results = self.engine.search(*search_args)
       rescue Exception => e
-        warn e
+        Rails.logger.error("\nBentoSearch:MultiSearcher caught exception: #{e}\n#{e.backtrace.join("   \n")}")
         # Make a fake results with caught exception. 
         @results = BentoSearch::Results.new
         @results.error ||= {}

data/app/models/bento_search/result_item.rb

@@ -3,6 +3,11 @@ module BentoSearch
   # with common data fields. Usually held in a BentoSearch::Results object.
   #
   # ANY field can be nil, clients should be aware.  
+  #
+  # Each item has a field for one main link as string url, at #link (which may be nil),
+  # as well as array of possibly additional links (with labels and metadata)
+  # under #other_links.  #other_links is an array of BentoSearch::Link 
+  # objects. 
   class ResultItem
     include ERB::Util # for html_escape for our presentational stuff
     include ActionView::Helpers::OutputSafetyHelper # for safe_join

data/app/models/bento_search/results.rb

@@ -30,8 +30,13 @@ module BentoSearch
     #     Possibly a ruby exception object. may be nil. 
     attr_accessor :error
     
-    # time it took to do search, in seconds. 
+    # time it took to do search, in seconds as float 
     attr_accessor :timing
+    # timing from #timing, but in miliseconds as int
+    def timing_ms
+      return nil if timing.nil?
+      (timing * 1000).to_i
+    end
     
     # search arguments as normalized by SearchEngine, not neccesarily
     # directly as input. A hash. 

data/app/models/bento_search/search_engine.rb

@@ -2,46 +2,117 @@ require 'active_support/concern'
 require 'active_support/core_ext/module/delegation'
 require 'confstruct'
 
+# just so we can catch their exceptions:
+require 'httpclient'
+require 'multi_json'
+require 'nokogiri'
 
 module BentoSearch
   # Module mix-in for bento_search search engines. 
   #
   # ==Using a SearchEngine 
   #
-  # * init/config
-  # * search
-  #   * pagination, with max per_page
-  #   * search fields, with semantics. ask for supported search fields. 
+  # See a whole bunch more examples in the project README.
   #
-  # == Standard config
-  #  * item_decorators : Array of Modules that will be decorated. See Decorators section. 
+  # You can initialize a search engine with configuration (some engines
+  # have required configuration):
+  #
+  #      engine = SomeSearchEngine.new(:config_key => 'foo')
+  #
+  # Or, it can be convenient (and is required for some features) to store
+  # a search engine with configuration in a global registry:
+  #
+  #      BentoSearch.register_engine("some_searcher") do |config|
+  #         config.engine = "SomeSearchEngine"
+  #         config.config_key = "foo"
+  #      end
+  #      # instantiates a new engine with registered config:
+  #      engine = BentoSearch.get_engine("some_searcher")
+  #
+  #  You can then use the #search method, which returns an instance of
+  #  of BentoSearch::Results
+  #
+  #  results = engine.search("query")
+  # 
+  #  See more docs under #search, as well as project README.  
+  #
+  # == Standard configuration variables. 
+  # 
+  # Some engines require their own engine-specific configuration for api keys
+  # and such, and offer their own engine-specific configuration for engine-specific
+  # features. 
+  #
+  # An additional semi-standard configuration variable, some engines take
+  # an `:auth => true` to tell the engine to assume that all access is by
+  # authenticated local users who should be given elevated access to results.  
+  #
+  # Additional standard configuration keys that are implemented by the bento_search
+  # framework:
+  #
+  #  [item_decorators]
+  #      Array of Modules that will be decorated on to each individual search
+  #      BentoSearch::ResultItem. These can be used to, via configuration, change
+  #      the links associated with items, change certain item behaviors, or massage
+  #      item metadata. (Needs more documentation). 
+  #      
   #
   # == Implementing a SearchEngine
   #
-  # `include BentoSearch::SearchEngine`
+  # Implmeneting a new SearchEngine is relatively straightforward -- you are
+  # generally only responsible for the parts specific to your search engine:
+  # receiving a query, making a call to the external search engine, and
+  # translating it's result to standard a BentoSearch::Results full of
+  # BentoSearch::ResultItems. 
+  #
+  # Start out by simply including the search engine module:
+  #
+  # class MyEngine
+  #   include BentoSearch::SearchEngine
+  # end
+  #
+  # Next, at a minimum, you need to implement a #search_implementation
+  # method, which takes a _normalized_ hash of search instructions as input
+  # (see documentation at #normalized_search_arguments), and returns
+  # BentoSearch::Results item.
   #
-  #  a SearchEngine's state should not be search-specific, but
-  #  is configuration specific. Don't store anything specific
-  #  to a specific search in iVars. 
+  # The Results object should have #total_items set with total hitcount, and
+  # contain BentoSearch::ResultItem objects for each hit in the current page. 
+  # See individual class documentation for more info. 
   #
-  #  Do implement `#search(*args)`
+  # That's about the extent of your responsibilities. If the search failed
+  # for some reason due to an error, you should return a Results object
+  # with it's #error object set, so it will be `failed?`.  The framework
+  # will take care of this for you for certain uncaught exceptions you allow
+  # to rise out of #search_implementation (timeouts, HTTPClient timeouts,
+  # nokogiri and MultiJson parse errors). 
+  #
+  # A SearchEngine object can be re-used for multiple searches, possibly
+  # under concurrent multi-threading. Do not store search-specific state
+  # in the search object. but you can store configuration-specific state there
+  # of course. 
   # 
-  #  Do use HTTPClient, if possible, for http searches, 
-  #  using a class-level HTTPClient to maintain persistent connections. 
+  # Recommend use of HTTPClient, if possible, for http searches. Especially
+  # using a class-level HTTPClient instance, to re-use persistent http
+  # connections accross searches (can be esp important if you need to contact
+  # external search api via https/ssl).
+  #
+  # If you have required configuration keys, you can register that with 
+  # class-level required_configuration_keys method. 
   #
-  #  Other options:
-  #  * implement a class-level `self.required_configuration' returning
-  #    an array of config keys or dot keypaths, and it'll raise on init
-  #    if those config's weren't supplied. 
-  #  * max per page
-  #  * search fields
+  # You can also advertise max per-page value by overriding max_per_page. 
   #
-  #  Some engines support `:auth => true` for elevated access to affiliated
-  #  users. 
+  # If you support fielded searching, you should over-ride 
+  # #search_field_definitions; if you support sorting, you should
+  # override #sort_definitions. See BentoSearch::SearchEngine::Capabilities
+  # module for documentation. 
+  # 
   #
   module SearchEngine
     DefaultPerPage = 10
     
+
+    
+    
     extend ActiveSupport::Concern
     
     include Capabilities
@@ -66,10 +137,10 @@ module BentoSearch
       # global defaults?
       self.configuration[:item_decorators] ||= []
             
-      # check for required keys
+      # check for required keys -- have to be present, and not nil
       if self.class.required_configuration
         self.class.required_configuration.each do |required_key|          
-          if self.configuration.lookup!(required_key.to_s, "**NOT_FOUND**") == "**NOT_FOUND**"
+          if ["**NOT_FOUND**", nil].include? self.configuration.lookup!(required_key.to_s, "**NOT_FOUND**")
             raise ArgumentError.new("#{self.class.name} requires configuration key #{required_key}")
           end
         end
@@ -77,9 +148,49 @@ module BentoSearch
       
     end
     
-    # Calls individual engine #search_implementation.
-    # first normalizes arguments, also adds on standard metadata
-    # to results. 
+    
+    # Method used to actually get results from a search engine.  
+    #
+    # When implementing a search engine, you do not override this #search
+    # method, but instead override #search_implementation. #search will
+    # call your specific #search_implementation, first normalizing the query
+    # arguments, and then normalizing and adding standard metadata to your return value.      
+    #
+    #  Most engines support pagination, sorting, and searching in a specific
+    #  field. 
+    #
+    #      # 1-based page index
+    #      engine.search("query", :per_page => 20, :page => 5)
+    #      # or use 0-based per-record index, engines that don't
+    #      # support this will round to nearest page. 
+    #      engine.search("query", :start => 20)
+    #
+    #  You can ask an engine what search fields it supports with engine.search_keys
+    #      engine.search("query", :search_field => "engine_search_field_name")
+    #
+    #  There are also normalized 'semantic' names you can use accross engines
+    #  (if they support them): :title, :author, :subject, maybe more. 
+    #
+    #      engine.search("query", :semantic_search_field => :title)
+    #
+    #  Ask an engine what semantic field names it supports with `engine.semantic_search_keys`
+    #
+    #  Ask an engine what sort fields it supports with `engine.sort_keys`. See
+    #  list of standard sort keys in I18n file at ./config/locales/en.yml, in
+    #  `en.bento_search.sort_keys`. 
+    #
+    #      engine.search("query", :sort => "some_sort_key")
+    #
+    #  Some engines support additional arguments to 'search', see individual
+    #  engine documentation. For instance, some engines support `:auth => true`
+    #  to give the user elevated search privileges when you have an authenticated
+    #  local user. 
+    #
+    # Query as first arg is just a convenience, you can also use a single hash
+    # argument. 
+    #
+    #      engine.search(:query => "query", :per_page => 20, :page => 4)
+    #
     def search(*arguments)
       start_t = Time.now
       
@@ -99,10 +210,39 @@ module BentoSearch
       results.timing = (Time.now - start_t)
         
       return results
+    rescue *auto_rescue_exceptions => e
+      # Uncaught exception, log and turn into failed Results object. We
+      # only catch certain types of exceptions, or it makes dev really
+      # confusing eating exceptions. This is intentionally a convenience
+      # to allow search engine implementations to just raise the exception
+      # and we'll turn it into a proper error. 
+      cleaned_backtrace = Rails.backtrace_cleaner.clean(e.backtrace)
+      log_msg = "BentoSearch::SearchEngine failed results: #{e.inspect}\n    #{cleaned_backtrace.join("\n    ")}"
+      Rails.logger.error log_msg
+      
+      failed = BentoSearch::Results.new
+      failed.error ||= {}
+      failed.error[:exception] = e
+      return failed
     end
         
 
-    
+    # Take the arguments passed into #search, which can be flexibly given
+    # in several ways, and normalize to an expected single hash that
+    # will be passed to an engine's #search_implementation. The output
+    # of this method is a single hash, and is what a #search_implementation
+    # can expect to receive as an argument, with keys: 
+    #
+    # [:query]        the query
+    # [:per_page]     will _always_ be present, using the default per_page if
+    #                 none given by caller
+    # [:start, :page] both :start and :page will _always_ be present, regardless
+    #                 of which the caller used. They will both be integers, even if strings passed in.
+    # [:search_field] A search field from the engine's #search_field_definitions, as string.  
+    #                 Even if the caller used :semantic_search_field, it'll be normalized
+    #                 to the actual local search_field key on output. 
+    # [:sort]         Sort key. 
+    #
     def normalized_search_arguments(*orig_arguments)
       arguments = {}
       
@@ -163,7 +303,7 @@ module BentoSearch
     alias_method :parse_search_arguments, :normalized_search_arguments
     
     
-
+    
    
     
     protected
@@ -177,6 +317,23 @@ module BentoSearch
       end
     end
     
+    # What exceptions should our #search wrapper rescue and turn
+    # into failed results instead of fatal errors? 
+    #
+    # Can't rescue everything, or we eat VCR/webmock errors, and lots
+    # of other errors we don't want to eat either, making
+    # development really confusing.  Perhaps could set this
+    # to be something diff in production and dev?
+    #
+    # This default list is probably useful already, but individual
+    # engines can override if it's convenient for their own error
+    # handling. 
+    def auto_rescue_exceptions
+      [TimeoutError, HTTPClient::TimeoutError, 
+            HTTPClient::ConfigurationError, HTTPClient::BadResponseError,
+            MultiJson::DecodeError, Nokogiri::SyntaxError]
+    end
+    
     
     module ClassMethods
       

data/app/models/bento_search/search_engine/capabilities.rb

@@ -1,4 +1,3 @@
-
 # Methods that describe a search engine's capabilities,
 # mixed into SearchEngine. Individual engine implementations
 # will often over-ride some or all of these methods. 
@@ -24,6 +23,7 @@ module BentoSearch::SearchEngine::Capabilities
       # Keys should where possible be _standard_ keys chosen from
       # those listed in config/i18n/en:bento_search.sort_keys.*
       # But if you need something not there, it can be custom to engine.
+      # (or we can add it there?). 
       # Value of hash is for internal use by engine, it may be a convenient
       # place to store implementation details. 
       #
@@ -32,6 +32,12 @@ module BentoSearch::SearchEngine::Capabilities
       def sort_definitions
         {}
       end
+      
+      # convenience to get just the sort keys, which is what client
+      # cares about. 
+      def sort_keys
+        sort_definitions.keys
+      end
                   
       # Override to return int max per-page. 
       def max_per_page

data/app/search_engines/bento_search/ebsco_host_engine.rb

@@ -58,6 +58,8 @@ require 'httpclient'
 # * The 'info' service can be used to see what databases you have access to. 
 # * DTD of XML Response, hard to interpret but all we've got: http://support.ebsco.com/eit/docs/DTD_EIT_WS_searchResponse.zip
 #
+#  Hard to find docs page on embedding EBSCO limiters (like peer reviewed only "RV Y") in search query: 
+#     http://eit.ebscohost.com/Pages/MethodDescription.aspx?service=~/Services/SearchService.asmx&method=Info
 #
 # 
 #
@@ -172,7 +174,7 @@ class BentoSearch::EbscoHostEngine
     components.uniq! # no need to have the same thing twice
     
     # some hard-coded cases for better user-displayable string
-    if components.first == "Academic Journal" && components.last == "Article"
+    if ["Academic Journal", "Journal"].include?(components.first) && ["Article", "Journal Article"].include?(components.last)
       return "Journal Article"
     elsif components.first == "Periodical" && components.length > 1
       return components.last

data/app/search_engines/bento_search/eds_engine.rb

@@ -39,10 +39,14 @@ require 'http_client_patch/include_client'
 # ourselves. However, in our testing, the first/only CustomLink was an
 # an OpenURL. If configuration.assume_first_custom_link_openurl is
 # true (as is default), it will be used to create an OpenURL link. However, in
-# our testing, many records don't have this at all. **Note** Ask EBSCO support
+# our testing, many records don't have this at all. **Note** You want 
 # to configure your profile so OpenURLs are ALWAYS included for all records, not
 # just records with no EBSCO fulltext, to ensure bento_search can get the
-# openurl. 
+# openurl. http://support.ebsco.com/knowledge_base/detail.php?id=1111 (May
+# have to ask EBSCO support for help, it's confusing!). 
+#
+# TODO: May have to add configuration code to pull the OpenURL link out by
+# it's configured name or label, not assume first one is it. 
 #
 # As always, you can customize links and other_links with Item Decorators. 
 #
@@ -83,6 +87,12 @@ require 'http_client_patch/include_client'
 # By default the engine will search as 'guest' unauth user. But config
 # 'auth' key to true to force all searches to auth (if you are protecting your
 # app) or pass :auth => true as param into #search method.  
+#
+# == Source Types
+# # What the EBSCO 'source types' mean: http://suprpot.ebsco.com/knowledge_base/detail.php?id=5382
+#
+# But "Dissertations" not "Dissertations/Theses". "Music Scores" not "Music Score". 
+
 #
 # == EDS docs:
 # 
@@ -90,6 +100,7 @@ require 'http_client_patch/include_client'
 # * EDS Wiki: http://edswiki.ebscohost.com/EDS_API_Documentation
 # * You'll need to request an account to the EDS wiki, see: http://support.ebsco.com/knowledge_base/detail.php?id=5990
 # 
+
 class BentoSearch::EdsEngine
   include BentoSearch::SearchEngine
   
@@ -152,7 +163,7 @@ class BentoSearch::EdsEngine
     end
     # Can't have any commas in query, it turns out, although
     # this is not documented. 
-    query += args[:query].gsub("/\,/", "")
+    query += args[:query].gsub(",", " ")
     
     url = "#{configuration.base_url}search?view=detailed&query=#{CGI.escape query}"
     
@@ -174,6 +185,11 @@ class BentoSearch::EdsEngine
       end
     end
     
+    if configuration.only_source_types.present?
+      # facetfilter=1,SourceType:Research Starters,SourceType:Books
+      url += "&facetfilter=" + CGI.escape("1," + configuration.only_source_types.collect {|t| "SourceType:#{t}"}.join(","))
+    end
+    
     
     return url
   end
@@ -189,7 +205,9 @@ class BentoSearch::EdsEngine
       with_session(end_user_auth) do |session_token|
                         
         url = construct_search_url(args)
-                
+             
+        
+        
         response = get_with_auth(url, session_token)
         
         results = BentoSearch::Results.new
@@ -202,6 +220,7 @@ class BentoSearch::EdsEngine
           item = BentoSearch::ResultItem.new
           
           item.title = prepare_eds_payload( element_by_group(record_xml, "Ti"), true )
+          
           if item.title.nil? && ! end_user_auth
             item.title = I18n.translate("bento_search.eds.record_not_available")
           end
@@ -213,9 +232,16 @@ class BentoSearch::EdsEngine
           # actual authors out of. WTF. Thanks for handling fragments
           # nokogiri. 
           author_mess = element_by_group(record_xml, "Au")
+          # only SOMETIMES does it have XML tags, other times it's straight text.
+          # ARGH.           
           author_xml = Nokogiri::XML::fragment(author_mess)
-          author_xml.xpath(".//searchLink").each do |author_node|
-            item.authors << BentoSearch::Author.new(:display => author_node.text)
+          searchLinks = author_xml.xpath(".//searchLink")
+          if searchLinks.size > 0
+            author_xml.xpath(".//searchLink").each do |author_node|
+              item.authors << BentoSearch::Author.new(:display => author_node.text)
+            end
+          else
+            item.authors << BentoSearch::Author.new(:display => author_xml.text)
           end
           
           
@@ -259,10 +285,17 @@ class BentoSearch::EdsEngine
           # tags in this mess, they will be lost. Probably don't
           # need highlighting in source anyhow. 
           citation_mess = element_by_group(record_xml, "Src")
-          citation_txt = Nokogiri::XML::fragment(citation_mess).text
-          # But strip off some "count of references" often on the end
-          # which are confusing and useless. 
-          item.custom_data["citation_blob"] = citation_txt.gsub(/ref +\d+ +ref\.$/, '')           
+          # Argh, but sometimes it's in SrcInfo _without_ tags instead
+          if citation_mess          
+            citation_txt = Nokogiri::XML::fragment(citation_mess).text
+            # But strip off some "count of references" often on the end
+            # which are confusing and useless. 
+            item.custom_data["citation_blob"] = citation_txt.gsub(/ref +\d+ +ref\.$/, '')
+          else
+            # try another location
+            item.custom_data["citation_blob"] = element_by_group(record_xml, "SrcInfo")
+          end
+          
                               
           item.extend CitationMessDecorator