bento_search 1.5.0 → 2.0.0.rc1

data/app/models/bento_search/concurrent_searcher.rb

@@ -0,0 +1,136 @@
+begin
+  require 'concurrent'
+
+  # Concurrently runs multiple searches in separate threads. Since a search
+  # generally spends most of it's time waiting on foreign API, this is
+  # useful to significantly reduce total latency of running multiple searches,
+  # even in MRI.
+  #
+  # Uses [concurrent-ruby](https://github.com/ruby-concurrency/concurrent-ruby),
+  # already a dependency of Rails 5.x. To use with Rails previous to 5.x,
+  # just add concurrent-ruby to your `Gemfile`:
+  #
+  #     gem 'concurrent-ruby', '~> 1.0'
+  #
+  # # Usage
+  #
+  # initialize with id's of registered engines:
+  #
+  #     searcher = BentoBox::ConcurrentSearcher.new(:gbs, :scopus)
+  #
+  # start the concurrent searches, params same as engine.search
+  #
+  #     searcher.search( query_params )
+  #
+  # retrieve results, blocking until all are completed:
+  #
+  #     results = searcher.results
+  #
+  # returns a Hash keyed by engine id, values BentoSearch::Results objects.
+  #
+  #     results # => { "gbs" => <BentoSearch::Results ...>, "scopus" =>  <BentoSearch::Results ...>}
+  #
+  # Calling results more than once will just return the initial results again
+  # (cached), it won't run a search again.
+  #
+  # ## Dev-mode autoloading and concurrency
+  #
+  # In Rails previous to Rails5, you may have to set config.cache_classes=true
+  # even in development to avoid problems. In Rails 5.x, we take advantage of
+  # new api that should allow concurrency-safe autoloading. But if you run into
+  # any weird problems (such as a deadlock), `cache_classes = true` and
+  # `eager_load = true` should eliminate them, at the cost of dev-mode
+  # auto-reloading.
+  #
+  #
+  # TODO: have a method that returns Futures instead of only supplying the blocking
+  # results method? Several tricks, including making sure to properly terminate actors.
+  class BentoSearch::ConcurrentSearcher
+    def initialize(*engine_ids)
+      auto_rescued_exceptions = [StandardError]
+
+      @engines = []
+      engine_ids.each do |id|
+        add_engine( BentoSearch.get_engine(id).tap { |e| e.auto_rescued_exceptions = auto_rescued_exceptions + e.auto_rescued_exceptions })
+      end
+      @extra_auto_rescue_exceptions = [StandardError]
+    end
+
+    # Adds an instantiated engine directly, rather than by id from global
+    # registry.
+    def add_engine(engine)
+      unless engine.configuration.id.present?
+        raise ArgumentError.new("ConcurrentSearcher engines need `configuration.id`, this one didn't have one: #{engine}")
+      end
+      @engines << engine
+    end
+
+    # Starts all searches, returns self so you can chain method calls if you like.
+    def search(*search_args)
+      search_args.freeze
+      @futures = @engines.collect do |engine|
+        Concurrent::Future.execute { rails_future_wrap { engine.search(*search_args) } }
+      end
+      return self
+    end
+
+    # Have you called #search yet? You can only call #results if you have.
+    # Will stay true forever, it doesn't tell you if the search is done or not.
+    def search_started?
+      !! @futures
+    end
+
+    # Call after #search. Blocks until each included engine is finished
+    # then returns a Hash keyed by engine registered id, value is a
+    # BentoSearch::Results object.
+    #
+    # If called multiple times, returns the same results each time, does
+    # not re-run searches.
+    #
+    # It is an error to invoke without having previously called #search
+    def results
+      unless search_started?
+        raise ArgumentError, "Can't call ConcurrentSearcher#results before you have executed a #search"
+      end
+
+      @results ||= begin
+        pairs = rails_wait_wrap do
+          @futures.collect { |future| [future.value!.engine_id, future.value!] }
+        end
+        Hash[ pairs ].freeze
+      end
+    end
+
+    protected
+
+    # In Rails5, future body's need to be wrapped in an executor,
+    # to handle auto-loading right in dev-mode, among other things.
+    # Rails docs coming, see https://github.com/rails/rails/issues/26847
+    @@rails_has_executor = Rails.application.respond_to?(:executor)
+    def rails_future_wrap
+      if @@rails_has_executor
+        Rails.application.executor.wrap { yield }
+      else
+        yield
+      end
+    end
+
+    # In Rails5, if we are collecting from within an action method
+    # (ie the 'request loop'), as we usually will be, we need to
+    # give up the autoload lock. Rails docs coming, see https://github.com/rails/rails/issues/26847
+    @@rails_needs_interlock_permit = ActiveSupport::Dependencies.respond_to?(:interlock) &&
+      !(Rails.application.config.eager_load && Rails.application.config.cache_classes)
+    def rails_wait_wrap
+      if @@rails_needs_interlock_permit
+        ActiveSupport::Dependencies.interlock.permit_concurrent_loads { yield }
+      else
+        yield
+      end
+    end
+
+  end
+rescue LoadError
+  # you can use bento_search without celluloid, just not
+  # this class.
+  $stderr.puts "Tried but could not load BentoSearch::ConcurrentSearcher, concurrent-ruby not available!"
+end

data/app/models/bento_search/result_item.rb

@@ -36,12 +36,12 @@ module BentoSearch
     # search service it came from. May be alphanumeric. May be nil
     # for engines that don't support it.
     serializable_attr_accessor :unique_id
-    
+
 
     # If set to true, item will refuse to generate an openurl,
     # returning nil from #to_openurl or #openurl_kev
     serializable_attr_accessor :openurl_disabled
-    
+
 
     # Array (possibly empty) of BentoSearch::Link objects
     # representing additional links. Often SearchEngine's themselves
@@ -52,7 +52,7 @@ module BentoSearch
 
     # * dc.title
     # * schema.org CreativeWork: 'name'
-    serializable_attr_accessor :title    
+    serializable_attr_accessor :title
     # backwards compat, we used to have separate titles and subtitles
     alias_method :complete_title, :title
 
@@ -112,7 +112,7 @@ module BentoSearch
     #
     # Note: We're re-thinking this, might allow uncontrolled
     # in here instead.
-    serializable_attr_accessor :format    
+    serializable_attr_accessor :format
 
     # Translated from internal format vocab at #format. Outputs
     # eg http://schema.org/Book
@@ -137,7 +137,7 @@ module BentoSearch
     # uncontrolled presumably english-language format string.
     # if supplied will be used in display in place of controlled
     # format.
-    serializable_attr_accessor :format_str    
+    serializable_attr_accessor :format_str
 
     # Language of materials. Producer can set language_code to an ISO 639-1 (two
     # letter) or 639-3 (three letter) language code. If you do this, you don't
@@ -153,11 +153,11 @@ module BentoSearch
     # #language_iso_639_2 (either may be null), or #language_str for uncontrolled
     # string. If engine just sets one of these, internals take care of filling
     # out the others. r
-    serializable_attr_accessor :language_code    
+    serializable_attr_accessor :language_code
     attr_writer :language_str
     def language_str
-      @language_str || language_code.try do |code|
-        LanguageList::LanguageInfo.find(code).try do |lang_obj|
+      (@language_str ||= nil) || language_code.try do |code|
+        LanguageList::LanguageInfo.find(code.dup).try do |lang_obj|
           lang_obj.name
         end
       end
@@ -167,7 +167,10 @@ module BentoSearch
     # if available, otherwise from direct language_str if available and
     # possible.
     def language_obj
-      @language_obj ||= LanguageList::LanguageInfo.find( self.language_code || self.language_str )
+      @language_obj ||= begin
+        lookup = self.language_code || self.language_str
+        LanguageList::LanguageInfo.find( lookup.dup ) if lookup
+      end
     end
 
     # Two letter ISO language code, or nil
@@ -230,12 +233,12 @@ module BentoSearch
 
     # An ARRAY of string query-in-context snippets. Will usually
     # have highlighting <b> tags in it. Creator is responsible
-    # for making sure it's otherwise html-safe. 
+    # for making sure it's otherwise html-safe.
     #
     # Not all engines may stores Snippets array in addition to abstract,
     # some may only store one or the other. Some may store both but
     # with same content formatted differently (array of multiple vs
-    # one combined string), some engines they may be different. 
+    # one combined string), some engines they may be different.
     attr_accessor :snippets
     serializable_attr :snippets
 
@@ -263,7 +266,7 @@ module BentoSearch
     # for it? Nice thing about the configuration has instead is it's
     # easily serializable, it's just data.
     #
-    # Although we intentionally do NOT include these in JSON serialization, ha. 
+    # Although we intentionally do NOT include these in JSON serialization, ha.
     attr_accessor :display_configuration
     attr_accessor :engine_id
 

data/app/models/bento_search/results/serialization.rb

@@ -4,21 +4,21 @@ require 'json'
 require 'date'
 
 # Call #dump_to_json on a BentoSearch value object (such as BentoSearch::Result or ::Author)
-# to get it in Json 
+# to get it in Json
 #
 # Values marked with serializable_attr in BentoSearch::Result are
-# included in seralization. 
+# included in seralization.
 #
 # At present metadata and configuration are NOT serialized: #decorator, #display_configuration,
 # and #engine_id are not included in the serialization, so when loaded from serialization,
-# ResultItems will not have such things set. 
-# 
+# ResultItems will not have such things set.
+#
 # * Works by getting and setting instance variables directly, ignores getters/setters
 #
 # * This means decorated values are NOT included in serialization, the raw
 #   values are what is serialized. This is intended, we serialize internal
 #   state, not decoration which can be recreated. You should make sure the decorators you
-#   want are applied after de-serialization. 
+#   want are applied after de-serialization.
 #
 # * preserves html_safety status in serialization, by adding extra `_attr_htmlsafe: true` key/value
 #
@@ -31,22 +31,23 @@ module BentoSearch::Results::Serialization
     self._serializable_attr_options = {}
   end
 
+
   class_methods do
     # Just a macro to mark a property name serializable -- the name is
     # of an instance method that will be included in our serializations
-    # and de-serializations. 
+    # and de-serializations.
     #
     # Options:
     #   * collection_of: String fully qualified name of a class that is
     #       is also BentoSearch::Results::Serialization, the attribute
-    #       is an array of these. 
+    #       is an array of these.
     #   * serializer: String fully qualified class name of a serializer
     #        class that has a `dump` and a `load` for individual values,
     #        we just use it for Date now, see BentoSearch::Results::Serialization::Date
     def serializable_attr(symbol, options = nil)
       symbol = symbol.to_s
       self._serializable_attrs << symbol
-      if options 
+      if options
         self._serializable_attr_options[symbol] = options
       end
     end
@@ -67,14 +68,14 @@ module BentoSearch::Results::Serialization
 
 
         if _serializable_attr_options[key] && _serializable_attr_options[key][:collection_of]
-          klass = qualified_const_get(_serializable_attr_options[key][:collection_of])
+          klass = correct_const_get(_serializable_attr_options[key][:collection_of])
           value = value.collect do |item|
             klass.from_internal_state_hash(item)
           end
         end
 
         if _serializable_attr_options[key] && _serializable_attr_options[key][:serializer]
-          klass = qualified_const_get(_serializable_attr_options[key][:serializer])
+          klass = correct_const_get(_serializable_attr_options[key][:serializer])
           value = klass.load(value)
         end
 
@@ -92,18 +93,26 @@ module BentoSearch::Results::Serialization
       self.from_internal_state_hash( JSON.parse! json_str )
     end
 
+    def correct_const_get(str)
+      if Gem::Version.new(Rails.version) > Gem::Version.new('4.2.99')
+        const_get(str)
+      else
+        qualified_const_get(str)
+      end
+    end
+
   end
 
   def internal_state_hash
     hash = {}
     self._serializable_attrs.each do |accessor|
       accessor = accessor.to_s
-      value = self.instance_variable_get("@#{accessor}")
+      value = self.instance_variable_defined?("@#{accessor}") && self.instance_variable_get("@#{accessor}")
 
       next if value.blank?
 
       if _serializable_attr_options[accessor] && _serializable_attr_options[accessor][:serializer]
-        klass = self.class.qualified_const_get(_serializable_attr_options[accessor][:serializer])
+        klass = self.class.correct_const_get(_serializable_attr_options[accessor][:serializer])
         value = klass.dump(value)
       elsif value.respond_to?(:to_ary)
         value = value.to_ary.collect do |item|
@@ -133,4 +142,4 @@ module BentoSearch::Results::Serialization
     end
   end
 
-end
+end

data/app/models/bento_search/search_engine.rb

@@ -9,18 +9,17 @@ require 'nokogiri'
 
 module BentoSearch
   # Usually raised by #get on an engine, when result for specified identifier
-  # can't be found. 
+  # can't be found.
   class ::BentoSearch::NotFound < ::BentoSearch::Error ; end
-  # Usually raised by #get when identifier results in more than one record. 
+  # Usually raised by #get when identifier results in more than one record.
   class ::BentoSearch::TooManyFound < ::BentoSearch::Error ; end
   # Raised for problem contacting or unexpected response from
-  # remote service. Not yet universally used. 
+  # remote service. Not yet universally used.
   class ::BentoSearch::FetchError < ::BentoSearch::Error ; end
 
-  
-  # Module mix-in for bento_search search engines. 
+  # Module mix-in for bento_search search engines.
   #
-  # ==Using a SearchEngine 
+  # ==Using a SearchEngine
   #
   # See a whole bunch more examples in the project README.
   #
@@ -43,18 +42,18 @@ module BentoSearch
   #  of BentoSearch::Results
   #
   #  results = engine.search("query")
-  # 
-  #  See more docs under #search, as well as project README.  
   #
-  # == Standard configuration variables. 
-  # 
+  #  See more docs under #search, as well as project README.
+  #
+  # == Standard configuration variables.
+  #
   # Some engines require their own engine-specific configuration for api keys
   # and such, and offer their own engine-specific configuration for engine-specific
-  # features. 
+  # features.
   #
   # An additional semi-standard configuration variable, some engines take
   # an `:auth => true` to tell the engine to assume that all access is by
-  # authenticated local users who should be given elevated access to results.  
+  # authenticated local users who should be given elevated access to results.
   #
   # Additional standard configuration keys that are implemented by the bento_search
   # framework:
@@ -63,7 +62,12 @@ module BentoSearch
   #      String name of decorator class that will be applied by #bento_decorate
   #      helper in standard view. See wiki for more info on decorators. Must be
   #      string name, actual class object not supported (to make it easier
-  #      to serialize and transport configuration). 
+  #      to serialize and transport configuration).
+  #
+  # [log_failed_results]
+  #       Default false, if true all failed results are logged to
+  #       `Rails.log.error`. Can set global default with
+  #        `BentoSearch.defaults.log_failed_results = true`
   #
   # == Implementing a SearchEngine
   #
@@ -71,7 +75,7 @@ module BentoSearch
   # generally only responsible for the parts specific to your search engine:
   # receiving a query, making a call to the external search engine, and
   # translating it's result to standard a BentoSearch::Results full of
-  # BentoSearch::ResultItems. 
+  # BentoSearch::ResultItems.
   #
   # Start out by simply including the search engine module:
   #
@@ -85,64 +89,102 @@ module BentoSearch
   # BentoSearch::Results item.
   #
   # The Results object should have #total_items set with total hitcount, and
-  # contain BentoSearch::ResultItem objects for each hit in the current page. 
-  # See individual class documentation for more info. 
+  # contain BentoSearch::ResultItem objects for each hit in the current page.
+  # See individual class documentation for more info.
   #
   # That's about the extent of your responsibilities. If the search failed
   # for some reason due to an error, you should return a Results object
   # with it's #error object set, so it will be `failed?`.  The framework
   # will take care of this for you for certain uncaught exceptions you allow
   # to rise out of #search_implementation (timeouts, HTTPClient timeouts,
-  # nokogiri and MultiJson parse errors). 
+  # nokogiri and MultiJson parse errors).
   #
   # A SearchEngine object can be re-used for multiple searches, possibly
   # under concurrent multi-threading. Do not store search-specific state
   # in the search object. but you can store configuration-specific state there
-  # of course. 
-  # 
+  # of course.
+  #
   # Recommend use of HTTPClient, if possible, for http searches. Especially
   # using a class-level HTTPClient instance, to re-use persistent http
   # connections accross searches (can be esp important if you need to contact
   # external search api via https/ssl).
   #
-  # If you have required configuration keys, you can register that with 
-  # class-level required_configuration_keys method. 
+  # If you have required configuration keys, you can register that with
+  # class-level required_configuration_keys method.
   #
-  # You can also advertise max per-page value by overriding max_per_page. 
+  # You can also advertise max per-page value by overriding max_per_page.
   #
-  # If you support fielded searching, you should over-ride 
+  # If you support fielded searching, you should over-ride
   # #search_field_definitions; if you support sorting, you should
   # override #sort_definitions. See BentoSearch::SearchEngine::Capabilities
-  # module for documentation. 
-  # 
+  # module for documentation.
+  #
   #
   module SearchEngine
     DefaultPerPage = 10
-    
 
-    
-    
     extend ActiveSupport::Concern
-    
+
     include Capabilities
-    
+
+    mattr_accessor :default_auto_rescued_exceptions
+    self.default_auto_rescued_exceptions = [
+        BentoSearch::RubyTimeoutClass,
+        HTTPClient::TimeoutError,
+        HTTPClient::ConfigurationError,
+        HTTPClient::BadResponseError,
+        MultiJson::DecodeError,
+        Nokogiri::SyntaxError,
+        SocketError
+      ].freeze
+
     included do
-      attr_accessor :configuration      
+      attr_accessor :configuration
+
+      # What exceptions should our #search wrapper rescue and turn
+      # into failed results instead of fatal errors?
+      #
+      # Can't rescue everything, or we eat VCR/webmock errors, and lots
+      # of other errors we don't want to eat either, making
+      # development really confusing.  Perhaps could set this
+      # to be something diff in production and dev?
+      #
+      # This default list is probably useful already, but individual
+      # engines can override if it's convenient for their own error
+      # handling.
+      #
+      # Override by just using `auto_rescued_exceptions=` on class _or_ method,
+      # although some legacy code may override `def auto_rescue_exceptions` (note
+      # old `rescue` vs new `rescued`) which should work too.
+      self.class_attribute :auto_rescued_exceptions
+      self.auto_rescued_exceptions = ::BentoSearch::SearchEngine.default_auto_rescued_exceptions
+
+      # Over-ride returning a hash or Confstruct with
+      # any configuration values you want by default.
+      # actual user-specified config values will be deep-merged
+      # into the defaults.
+      def self.default_configuration
+      end
+
+      # Over-ride returning an array of symbols for required
+      # configuration keys.
+      def self.required_configuration
+      end
     end
-    
+
     # If specific SearchEngine calls initialize, you want to call super
     # handles configuration loading, mostly. Argument is a
-    # Confstruct::Configuration or Hash. 
+    # Confstruct::Configuration or Hash.
     def initialize(aConfiguration = Confstruct::Configuration.new)
       # To work around weird confstruct bug, we need to change
-      # a hash to a Confstruct ourselves. 
+      # a hash to a Confstruct ourselves.
       # https://github.com/mbklein/confstruct/issues/14
       unless aConfiguration.kind_of? Confstruct::Configuration
         aConfiguration = Confstruct::Configuration.new aConfiguration
       end
-        
-      
-      # init, from copy of default, or new      
+
+
+      # init, from copy of default, or new
       if self.class.default_configuration
         self.configuration = Confstruct::Configuration.new(self.class.default_configuration)
       else
@@ -150,187 +192,193 @@ module BentoSearch
       end
       # merge in current instance config
       self.configuration.configure ( aConfiguration )
-      
-      # global defaults?      
+
+      # global defaults?
       self.configuration[:for_display] ||= {}
-            
+      unless self.configuration.has_key?(:log_failed_results)
+        self.configuration[:log_failed_results] = BentoSearch.defaults.log_failed_results
+      end
+
       # check for required keys -- have to be present, and not nil
       if self.class.required_configuration
-        self.class.required_configuration.each do |required_key|          
+        self.class.required_configuration.each do |required_key|
           if ["**NOT_FOUND**", nil].include? self.configuration.lookup!(required_key.to_s, "**NOT_FOUND**")
             raise ArgumentError.new("#{self.class.name} requires configuration key #{required_key}")
           end
         end
       end
-      
+
     end
-    
-    
-    # Method used to actually get results from a search engine.  
+
+
+    # Method used to actually get results from a search engine.
     #
     # When implementing a search engine, you do not override this #search
     # method, but instead override #search_implementation. #search will
     # call your specific #search_implementation, first normalizing the query
-    # arguments, and then normalizing and adding standard metadata to your return value.      
+    # arguments, and then normalizing and adding standard metadata to your return value.
     #
     #  Most engines support pagination, sorting, and searching in a specific
-    #  field. 
+    #  field.
     #
     #      # 1-based page index
     #      engine.search("query", :per_page => 20, :page => 5)
     #      # or use 0-based per-record index, engines that don't
-    #      # support this will round to nearest page. 
+    #      # support this will round to nearest page.
     #      engine.search("query", :start => 20)
     #
     #  You can ask an engine what search fields it supports with engine.search_keys
     #      engine.search("query", :search_field => "engine_search_field_name")
     #
     #  There are also normalized 'semantic' names you can use accross engines
-    #  (if they support them): :title, :author, :subject, maybe more. 
+    #  (if they support them): :title, :author, :subject, maybe more.
     #
     #      engine.search("query", :semantic_search_field => :title)
     #
     #  Ask an engine what semantic field names it supports with `engine.semantic_search_keys`
     #
-    #  Unrecognized search fields will be ignored, unless you pass in 
-    #  :unrecognized_search_field => :raise (or do same in config). 
+    #  Unrecognized search fields will be ignored, unless you pass in
+    #  :unrecognized_search_field => :raise (or do same in config).
     #
     #  Ask an engine what sort fields it supports with `engine.sort_keys`. See
     #  list of standard sort keys in I18n file at ./config/locales/en.yml, in
-    #  `en.bento_search.sort_keys`. 
+    #  `en.bento_search.sort_keys`.
     #
     #      engine.search("query", :sort => "some_sort_key")
     #
     #  Some engines support additional arguments to 'search', see individual
     #  engine documentation. For instance, some engines support `:auth => true`
     #  to give the user elevated search privileges when you have an authenticated
-    #  local user. 
+    #  local user.
     #
     # Query as first arg is just a convenience, you can also use a single hash
-    # argument. 
+    # argument.
     #
     #      engine.search(:query => "query", :per_page => 20, :page => 4)
     #
     def search(*arguments)
       start_t = Time.now
-      
+
       arguments = normalized_search_arguments(*arguments)
 
       results = search_implementation(arguments)
-      
+
       fill_in_search_metadata_for(results, arguments)
-            
+
       results.timing = (Time.now - start_t)
-                    
+
       return results
     rescue *auto_rescue_exceptions => e
       # Uncaught exception, log and turn into failed Results object. We
       # only catch certain types of exceptions, or it makes dev really
       # confusing eating exceptions. This is intentionally a convenience
       # to allow search engine implementations to just raise the exception
-      # and we'll turn it into a proper error. 
+      # and we'll turn it into a proper error.
       cleaned_backtrace = Rails.backtrace_cleaner.clean(e.backtrace)
       log_msg = "BentoSearch::SearchEngine failed results: #{e.inspect}\n    #{cleaned_backtrace.join("\n    ")}"
       Rails.logger.error log_msg
-      
+
       failed = BentoSearch::Results.new
       failed.error ||= {}
       failed.error[:exception] = e
-      
+
       failed.timing                = (Time.now - start_t)
-      
+
       fill_in_search_metadata_for(failed, arguments)
 
-      
       return failed
+    ensure
+      if results && configuration.log_failed_results && results.failed?
+        Rails.logger.error("Error fetching results for `#{configuration.id || self}`: #{arguments}: #{results.error}")
+      end
     end
-    
+
     # SOME of the elements of Results to be returned that SearchEngine implementation
     # fills in automatically post-search. Extracted into a method for DRY in
     # error handling to try to fill these in even in errors. Also can be used
-    # as public method for de-serialized or mock results. 
+    # as public method for de-serialized or mock results.
     def fill_in_search_metadata_for(results, normalized_arguments = {})
       results.search_args           = normalized_arguments
       results.start = normalized_arguments[:start] || 0
       results.per_page = normalized_arguments[:per_page]
-      
+
       results.engine_id             = configuration.id
       results.display_configuration = configuration.for_display
 
       # We copy some configuraton info over to each Item, as a convenience
       # to display logic that may have decide what to do given only an item,
       # and may want to parameterize based on configuration.
-      results.each do |item| 
-        item.engine_id              = configuration.id 
+      results.each do |item|
+        item.engine_id              = configuration.id
         item.decorator              = configuration.lookup!("for_display.decorator")
         item.display_configuration  = configuration.for_display
       end
 
       results
     end
-        
+
 
     # Take the arguments passed into #search, which can be flexibly given
     # in several ways, and normalize to an expected single hash that
     # will be passed to an engine's #search_implementation. The output
     # of this method is a single hash, and is what a #search_implementation
-    # can expect to receive as an argument, with keys: 
+    # can expect to receive as an argument, with keys:
     #
     # [:query]        the query
     # [:per_page]     will _always_ be present, using the default per_page if
     #                 none given by caller
     # [:start, :page] both :start and :page will _always_ be present, regardless
     #                 of which the caller used. They will both be integers, even if strings passed in.
-    # [:search_field] A search field from the engine's #search_field_definitions, as string.  
+    # [:search_field] A search field from the engine's #search_field_definitions, as string.
     #                 Even if the caller used :semantic_search_field, it'll be normalized
-    #                 to the actual local search_field key on output. 
-    # [:sort]         Sort key. 
+    #                 to the actual local search_field key on output.
+    # [:sort]         Sort key.
     #
     def normalized_search_arguments(*orig_arguments)
       arguments = {}
-      
+
       # Two-arg style to one hash, if present
       if (orig_arguments.length > 1 ||
           (orig_arguments.length == 1 && ! orig_arguments.first.kind_of?(Hash)))
-        arguments[:query] = orig_arguments.delete_at(0)      
+        arguments[:query] = orig_arguments.delete_at(0)
       end
 
       arguments.merge!(orig_arguments.first)  if orig_arguments.length > 0
-      
-      
+
+
       # allow strings for pagination (like from url query), change to
-      # int please. 
+      # int please.
       [:page, :per_page, :start].each do |key|
         arguments.delete(key) if arguments[key].blank?
         arguments[key] = arguments[key].to_i if arguments[key]
-      end   
-      arguments[:per_page] ||= DefaultPerPage
-      
-      # illegal arguments      
+      end
+      arguments[:per_page] ||= configuration.default_per_page || DefaultPerPage
+
+      # illegal arguments
       if (arguments[:start] && arguments[:page])
         raise ArgumentError.new("Can't supply both :page and :start")
       end
-      if ( arguments[:per_page] && 
-           self.max_per_page && 
+      if ( arguments[:per_page] &&
+           self.max_per_page &&
            arguments[:per_page] > self.max_per_page)
         raise ArgumentError.new("#{arguments[:per_page]} is more than maximum :per_page of #{self.max_per_page} for #{self.class}")
       end
-   
-      
+
+
       # Normalize :page to :start, and vice versa
       if arguments[:page]
         arguments[:start] = (arguments[:page] - 1) * arguments[:per_page]
       elsif arguments[:start]
         arguments[:page] = (arguments[:start] / arguments[:per_page]) + 1
       end
-      
+
       # normalize :sort from possibly symbol to string
       # TODO: raise if unrecognized sort key?
       if arguments[:sort]
         arguments[:sort] = arguments[:sort].to_s
       end
 
-      
+
       # Multi-field search
       if arguments[:query].kind_of? Hash
         # Only if allowed
@@ -348,7 +396,7 @@ module BentoSearch
         # translate semantic fields, raising for unfound fields if configured
         arguments[:query].transform_keys! do |key|
           new_key = self.semantic_search_map[key.to_s] || key
-          
+
           if ( config_arg(arguments, :unrecognized_search_field) == "raise" &&
               ! self.search_keys.include?(new_key))
             raise ArgumentError.new("#{self.class.name} does not know about search_field #{new_key}, in query Hash #{arguments[:query]}")
@@ -358,91 +406,73 @@ module BentoSearch
         end
 
       end
-      
+
       # translate semantic_search_field to search_field, or raise if
-      # can't. 
+      # can't.
       if (semantic = arguments.delete(:semantic_search_field)) && ! semantic.blank?
         semantic = semantic.to_s
         # Legacy publication_title is now called source_title
         semantic = "source_title" if semantic == "publication_title"
 
         mapped = self.semantic_search_map[semantic]
-        if config_arg(arguments, :unrecognized_search_field) == "raise" && ! mapped 
+        if config_arg(arguments, :unrecognized_search_field) == "raise" && ! mapped
           raise ArgumentError.new("#{self.class.name} does not know about :semantic_search_field #{semantic}")
         end
         arguments[:search_field] = mapped
-      end      
+      end
       if config_arg(arguments, :unrecognized_search_field) == "raise" && ! search_keys.include?(arguments[:search_field])
         raise ArgumentError.new("#{self.class.name} does not know about :search_field #{arguments[:search_field]}")
       end
-        
-              
+
+
       return arguments
     end
     alias_method :parse_search_arguments, :normalized_search_arguments
-    
-    
-    # Used mainly/only by the AJAX results loading. 
+
+
+    # Used mainly/only by the AJAX results loading.
     # an array WHITELIST of attributes that can be sent as non-verified
     # request params and used to execute a search. For instance, 'auth' is
-    # NOT on there, you can't trust a web request as to 'auth' status. 
+    # NOT on there, you can't trust a web request as to 'auth' status.
     # individual engines may over-ride, call super, and add additional
-    # engine-specific attributes. 
+    # engine-specific attributes.
     def public_settable_search_args
       [:query, :search_field, :semantic_search_field, :sort, :page, :start, :per_page]
     end
-   
-    
+
+    # Cover method for consistent api with Results
+    def display_configuration
+      configuration.for_display
+    end
+
+    # Cover method for consistent api with Results
+    def engine_id
+      configuration.id
+    end
+
+
     protected
 
+    # For legacy reasons old name auto_rescue_exceptions is here, some
+    # sub-classes may override it. Now preferred to use auto_rescued_exceptions
+    # setter instead.
+    def auto_rescue_exceptions
+      self.auto_rescued_exceptions
+    end
+
     # get value of an arg that can be supplied in search args OR config,
     # with search_args over-ridding config. Also normalizes value to_s
-    # (for symbols/strings). 
+    # (for symbols/strings).
     def config_arg(arguments, key, default = nil)
       value = if arguments[key].present?
         arguments[key]
       else
         configuration[key]
       end
-      
+
       value = value.to_s if value.kind_of? Symbol
-      
+
       return value
     end
-    
-    # What exceptions should our #search wrapper rescue and turn
-    # into failed results instead of fatal errors? 
-    #
-    # Can't rescue everything, or we eat VCR/webmock errors, and lots
-    # of other errors we don't want to eat either, making
-    # development really confusing.  Perhaps could set this
-    # to be something diff in production and dev?
-    #
-    # This default list is probably useful already, but individual
-    # engines can override if it's convenient for their own errorau
-    # handling. 
-    def auto_rescue_exceptions
-      [TimeoutError, HTTPClient::TimeoutError, 
-            HTTPClient::ConfigurationError, HTTPClient::BadResponseError,
-            MultiJson::DecodeError, Nokogiri::SyntaxError]
-    end
-    
-    
-    module ClassMethods
-      
-      # Over-ride returning a hash or Confstruct with 
-      # any configuration values you want by default. 
-      # actual user-specified config values will be deep-merged
-      # into the defaults. 
-      def default_configuration
-      end
-      
-      # Over-ride returning an array of symbols for required
-      # configuration keys.
-      def required_configuration
-      end
-      
-    end
-    
   end
 end