bento_search 0.0.1

data/app/models/bento_search/search_engine/capabilities.rb

@@ -0,0 +1,65 @@
+
+# Methods that describe a search engine's capabilities,
+# mixed into SearchEngine. Individual engine implementations
+# will often over-ride some or all of these methods. 
+module BentoSearch::SearchEngine::Capabilities
+      # If support fielded search, over-ride to specify fields
+      # supported. Returns a hash, key is engine-specific internal
+      # search field, value is nil or a hash of metadata about
+      # the search field, including semantic mapping. 
+      #
+      # def search_field_definitions
+      #  { "intitle" => {:semantic => :title}}
+      # end
+      def search_field_definitions
+        {}
+      end
+      
+      # Over-ride with a HASH of available sorts. Each key is the string
+      # that will be passed in engine.search(...., :sort => key)
+      # The key combines a choice of sort field, ascending/descending,
+      # secondary sorts etc -- we combine this all with one key, because
+      # typical examined interfaces did same from a select menu. 
+      #
+      # Keys should where possible be _standard_ keys chosen from
+      # those listed in config/i18n/en:bento_search.sort_keys.*
+      # But if you need something not there, it can be custom to engine.
+      # Value of hash is for internal use by engine, it may be a convenient
+      # place to store implementation details. 
+      #
+      # For a particular engine, a sort not mentioned here will-- raise?
+      # be ignored? Not sure. 
+      def sort_definitions
+        {}
+      end
+                  
+      # Override to return int max per-page. 
+      def max_per_page
+        nil
+      end
+      
+      # Returns list of string internal search_field's that can
+      # be supplied to search(:search_field => x)
+      def search_keys        
+        return search_field_definitions.keys
+      end
+      
+      # Returns list of symbol semantic_search_field that can be
+      # supplied to search(:semantic_search_field => x)
+      def semantic_search_keys  
+        semantic_search_map.keys
+      end
+      
+      # returns a hash keyed by semantic search field symbol,
+      # value string internal search field key. 
+      def semantic_search_map                
+        # Hash[] conveniently takes an array of k-v pairs. 
+        return Hash[
+          search_field_definitions.collect do |field, defn|
+            [ defn[:semantic].to_s, field ] if defn && defn[:semantic]
+          end.compact        
+        ]
+      end
+      
+
+end

data/app/search_engines/bento_search/#Untitled-1#

@@ -0,0 +1,11 @@
+http://blacklight.mse.jhu.edu:3001/resolve?url_ver=Z39.88-2004
+&url_ctx_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Actx
+&ctx_ver=Z39.88-2004
+&ctx_tim=2012-07-25T16%3A21%3A11-04%3A00
+&ctx_id=
+&ctx_enc=info%3Aofi%2Fenc%3AUTF-8
+&rft.title=Monkey+Brains
+&rft.creator=Will.i.am
+&rft.pub=Absolute+Pitch%2C12+Dec+2007
+&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Adc
+&rfr_id=info%3Asid%2Fsummon.serialssolutions.com

data/app/search_engines/bento_search/ebsco_host_engine.rb

@@ -0,0 +1,356 @@
+require 'nokogiri'
+
+require 'http_client_patch/include_client'
+require 'httpclient'
+
+# Right now for EbscoHost API (Ebsco Integration Toolkit/EIT), 
+# may be expanded or refactored for EDS too.
+#
+# == Required Configuration
+#
+# * profile_id
+# * profile_password
+# * databases: ARRAY of ebsco shortcodes of what databases to include in search. If you specify one you don't have access to, you get an error message from ebsco, alas. 
+#
+# == Note on including databases
+#
+# Need to specifically configure all databases your institution licenses from
+# EBSCO that you want included in the search. You can't just say "all of them"
+# the api doesn't support that, and also more than 30 or 40 starts getting
+# horribly slow. If you include a db you do not have access to, EBSCO api
+# fatal errors. 
+#
+# You may want to make sure all your licensed databases are included
+# in your EIT profile. Log onto ebscoadmin, Customize Services, choose
+# EIT profile, choose 'databases' tag. 
+# 
+# === Download databases from EBSCO api
+#
+# We include a utility to download ALL activated databases for EIT profile
+# and generate a file putting them in a ruby array. You may want to use this
+# file as a starting point, and edit by hand:
+#
+# First configure your EBSCO search engine with bento_search, say under
+# key 'ebscohost'. 
+#
+# Then run:
+#    rails generate bento_search:pull_ebsco_dbs ebscohost
+#
+# assuming 'ebscohost' is the key you registered the EBSCO search engine. 
+#
+# This will create a file at ./config/ebsco_dbs.rb. You may want to hand
+# edit it. Then, in your bento search config, you can:
+#
+#    require "#{Rails.root}/config/ebsco_dbs.rb"
+#    BentoSearch.register_engine("ebscohost") do |conf|
+#       # ....
+#       conf.databases = $ebsco_dbs
+#    end
+#
+# == Vendor documentation 
+#
+# Vendor documentation is a bit scattered, main page:
+# * http://support.ebsco.com/eit/ws.php
+# Some other useful pages we discovered:
+# * http://support.ebsco.com/eit/ws_faq.php
+# * search syntax examples: http://support.ebsco.com/eit/ws_howto_queries.php
+# * Try construct a query: http://eit.ebscohost.com/Pages/MethodDescription.aspx?service=/Services/SearchService.asmx&method=Search
+# * The 'info' service can be used to see what databases you have access to. 
+# * DTD of XML Response, hard to interpret but all we've got: http://support.ebsco.com/eit/docs/DTD_EIT_WS_searchResponse.zip
+#
+#
+# 
+#
+# TODO: David Walker tells us we need to configure in EBSCO to make default operator be 'and' instead of phrase search!
+# We Do need to do that to get reasonable results. 
+class BentoSearch::EbscoHostEngine
+  include BentoSearch::SearchEngine
+  
+  extend HTTPClientPatch::IncludeClient
+  include_http_client
+  
+  # Include some rails helpers, text_helper.trucate
+  def text_helper
+    @@truncate ||= begin
+      o = Object.new
+      o.extend ActionView::Helpers::TextHelper
+      o
+    end
+  end
+  
+  def search_implementation(args)
+    url = query_url(args)
+    
+    results = BentoSearch::Results.new
+    xml, response, exception = nil, nil, nil
+    
+    begin
+      response = http_client.get(url)
+      xml = Nokogiri::XML(response.body)
+    rescue TimeoutError, HTTPClient::ConfigurationError, HTTPClient::BadResponseError, Nokogiri::SyntaxError  => e
+        exception = e        
+    end
+    # error handle
+    if ( response.nil? || 
+         xml.nil? || 
+         exception || 
+         (! HTTP::Status.successful? response.status) ||
+         (fault = xml.at_xpath("./Fault")))
+    
+         results.error ||= {}
+         results.error[:exception] = exception if exception
+         results.error[:status] = response.status if response
+         
+         if fault
+           results.error[:error_info] = text_if_present fault.at_xpath("./Message")
+         end
+            
+         return results
+    end
+             
+    
+    
+    # the namespaces they provide are weird and don't help and sometimes
+    # not clearly even legal. Remove em!
+    xml.remove_namespaces!
+    
+    results.total_items = xml.at_xpath("./searchResponse/Hits").text.to_i
+    
+    xml.xpath("./searchResponse/SearchResults/records/rec").each do |xml_rec|
+      results << item_from_xml( xml_rec )
+    end
+    
+    return results
+    
+  end
+  
+  
+  # Pass in a nokogiri node, return node.text, or nil if
+  # arg was nil or node.text was blank?
+  def text_if_present(node)
+    if node.nil? || node.text.blank?
+      nil
+    else
+      node.text
+    end    
+  end
+  
+  # Figure out proper controlled format for an ebsco item. 
+  # EBSCOHost (not sure about EDS) publication/document type
+  # are totally unusable non-normalized vocabulary for controlled
+  # types, we'll try to guess from other metadata features.   
+  def sniff_format(xml_node)
+    return nil if xml_node.nil?
+    
+    if xml_node.at_xpath("./bkinfo/*")
+      "Book"
+    elsif xml_node.at_xpath("./dissinfo/*")
+      :dissertation
+    elsif xml_node.at_xpath("./jinfo/*") && xml_node.at_xpath("./artinfo/*")
+      "Article"
+    elsif xml_node.at_xpath("./jinfo/*")
+      :serial
+    else
+      nil
+    end    
+  end
+  
+  # Figure out uncontrolled literal string format to show to users.
+  # We're going to try combining Ebsco Publication Type and Document Type,
+  # when both are present. Then a few hard-coded special transformations. 
+  def sniff_format_str(xml_node)  
+    pubtype = text_if_present( xml_node.at_xpath("./artinfo/pubtype") )
+    doctype = text_if_present( xml_node.at_xpath("./artinfo/doctype") )
+    
+    components = []
+    components.push pubtype
+    components.push doctype unless doctype == pubtype
+    
+    components.compact!
+    
+    components = components.collect {|a| a.titlecase if a}
+    components.uniq! # no need to have the same thing twice
+    
+    # some hard-coded cases for better user-displayable string
+    if components.first == "Academic Journal" && components.last == "Article"
+      return "Journal Article"
+    elsif components.first == "Periodical" && components.length > 1
+      return components.last
+    end
+    
+    
+    
+    return components.join(": ")
+  end
+  
+  # pass in <rec> nokogiri, will determine best link
+  def get_link(xml)
+    text_if_present(xml.at_xpath("./pdfLink")) || text_if_present(xml.at_xpath("./plink") )
+  end
+  
+  
+  # it's unclear if ebsco API actually allows escaping of special chars,
+  # or what the special chars are. But we know parens are special, can't
+  # escape em, we'll just remove em (should not effect search). 
+  def ebsco_query_escape(txt)
+    txt.gsub(/[)(]/, ' ')
+  end
+  
+  # Actually turn the user's query into an EBSCO "AND" boolean query,
+  # seems only way to get decent results where terms can match cross-fields
+  # at the moment, for EIT. We'll see for EDS. 
+  def ebsco_query_prepare(txt)
+    # use string split with regex cleverly to split into space
+    # seperated terms and phrases, keeping phrases as unit. 
+    terms = txt.split %r{[[:space:]]+|("[^"]+")}
+
+    # Remove parens in non-phrase-quoted terms
+    terms = terms.collect do |t| 
+      (t =~ /^\".*\"$/) ? t : ebsco_query_escape(t)      
+    end
+    
+    # Remove boolean operators if they are bare not in a phrase, they'll
+    # make things weird. In phrase quotes they are okay. 
+    # Remove empty strings. Remove terms that are solely punctuation
+    # without any letters. 
+    terms.delete_if do |term|
+      ( 
+        term.blank? || 
+        ["AND", "OR", "NOT"].include?(term) ||
+        term =~ /\A[^[[:alnum:]]]+\Z/
+      )
+    end
+    
+    terms.join(" AND ")    
+  end
+  
+  def query_url(args)
+    
+    url = 
+      "#{configuration.base_url}/Search?prof=#{configuration.profile_id}&pwd=#{configuration.profile_password}"
+    
+    query = ebsco_query_prepare  args[:query]  
+    
+    # wrap in (FI $query) if fielded search
+    if args[:search_field]
+      query = "(#{args[:search_field]} #{query})"
+    end
+    
+    url += "&query=#{CGI.escape query}"
+    
+    # startrec is 1-based for ebsco, not 0-based like for us. 
+    url += "&startrec=#{args[:start] + 1}" if args[:start]
+    url += "&numrec=#{args[:per_page]}" if args[:per_page]
+    
+    # Make relevance our default sort, rather than EBSCO's date. 
+    args[:sort] ||= "relevance"
+    url += "&sort=#{ sort_definitions[args[:sort]][:implementation]}"
+    
+    # Contrary to docs, don't pass these comma-seperated, pass em in seperate
+    # query params. 
+    configuration.databases.each do |db|
+      url += "&db=#{db}"
+    end    
+    
+    return url
+  end
+  
+  # pass in a nokogiri representing an EBSCO <rec> result,
+  # we'll turn it into a BentoSearch::ResultItem. 
+  def item_from_xml(xml_rec)        
+    info = xml_rec.at_xpath("./header/controlInfo")
+    
+    item = BentoSearch::ResultItem.new
+    
+    item.link           = get_link(xml_rec)
+
+    item.issn           = text_if_present info.at_xpath("./jinfo/issn")
+    item.journal_title  =  text_if_present(info.at_xpath("./jinfo/jtl"))
+    item.publisher      = text_if_present info.at_xpath("./pubinfo/pub")
+    # Might have multiple ISBN's in record, just take first for now
+    item.isbn           = text_if_present info.at_xpath("./bkinfo/isbn")
+    
+    item.year           = text_if_present info.at_xpath("./pubinfo/dt/@year")
+    item.volume         = text_if_present info.at_xpath("./pubinfo/vid")
+    item.issue          = text_if_present info.at_xpath("./pubinfo/iid")
+    
+    # EBSCO sometimes has crazy long titles, truncate em.     
+    item.title          = text_helper.truncate( text_if_present( info.at_xpath("./artinfo/tig/atl") ), :length => 200)
+    item.start_page     = text_if_present info.at_xpath("./artinfo/ppf")
+    
+    item.doi            = text_if_present info.at_xpath("./artinfo/ui[@type='doi']")
+    
+    item.abstract       = text_if_present info.at_xpath("./artinfo/ab")
+    # EBSCO abstracts have an annoying habit of beginning with "Abstract:"
+    if item.abstract
+      item.abstract.gsub!(/^Abstract\: /, "")
+    end
+    
+    # authors, only get full display name from EBSCO. 
+    info.xpath("./artinfo/aug/au").each do |author|
+      a = BentoSearch::Author.new(:display => author.text)
+      item.authors << a
+    end
+   
+    
+    item.format         = sniff_format info
+    item.format_str     = sniff_format_str info
+    
+    
+    return item
+  end
+  
+  # This method is not used for normal searching, but can be used by
+  # other code to retrieve the results of the EBSCO API Info command, 
+  # using connection details configured in this engine. The Info command
+  # can tell you what databases your account is authorized to see.
+  # Returns the complete Nokogiri response, but WITH NAMESPACES REMOVED
+  def get_info
+    url = 
+      "#{configuration.base_url}/Info?prof=#{configuration.profile_id}&pwd=#{configuration.profile_password}"    
+    
+    noko = Nokogiri::XML( http_client.get( url ).body )
+    
+    noko.remove_namespaces!
+    
+    return noko
+  end
+  
+  # David Walker says pretty much only relevance and date are realiable
+  # in EBSCOhost cross-search. 
+  def sort_definitions
+    { 
+      "relevance" => {:implementation => "relevance"},
+      "date_desc" => {:implementation => "date"}
+    }      
+  end
+  
+  def search_field_definitions
+    {
+      "AU"    => {:semantic => :author},
+      "TI"    => {:semantic => :title},
+      "SU"    => {:semantic => :subject},
+      "IS"    => {:semantic => :issn},
+      "IB"    => {:semantic => :isbn}
+    }
+  end
+  
+  def max_per_page
+    # Actually only '50' if you ask for 'full' records, but I don't think
+    # we need to do that ever, that's actually getting fulltext back! 
+    200
+  end
+  
+  def self.required_configuration
+    ["profile_id", "profile_password"]
+  end
+  
+  def self.default_configuration
+    {
+      # /Search
+      :base_url => "http://eit.ebscohost.com/Services/SearchService.asmx",
+      :databases => []
+    }
+  end
+  
+end

data/app/search_engines/bento_search/eds_engine.rb

@@ -0,0 +1,557 @@
+# encoding: UTF-8
+
+require 'nokogiri'
+require 'httpclient'
+require 'multi_json'
+require 'http_client_patch/include_client'
+
+
+#
+# For EBSCO Discovery Service. You will need a license to use. 
+#
+# == Required Configuration
+#
+# user_id, password: As given be EBSCO for access to EDS API (may be an admin account in ebscoadmin? Not sure). 
+# profile: As given by EBSCO, might be "edsapi"?
+#
+# == Highlighting
+#
+# EDS has a query-in-context highlighting feature. It is used by defualt, set 
+# config 'highlighting' to false to disable. 
+# If turned on, you may get <b class="bento_search_highlight"> tags
+# in title and abstract output if it's on, marked html_safe.  
+#
+# If highlighting is on, since the abstract will be marked html safe, the
+# view layer won't be able to safely truncate it. In fact, it's very hard
+# to do here too, but we do it anyway, by default to approx configuration
+# truncate_highlighted num of chars (default 280). Set to nil if you don't
+# want this. 
+#
+# == Linking
+#
+# The link to record in EBSCO interface delivered as "PLink" will be listed
+# as record main link. 
+#
+# Any links listed under <CustomLinks> will be listed as other_links, using
+# configured name provided by EBSCO for CustomLink.
+#
+# EDS Response does not have sufficient metadata for us to generate an OpenURL
+# ourselves. However, in our testing, the first/only CustomLink was an
+# an OpenURL. If configuration.assume_first_custom_link_openurl is
+# true (as is default), it will be used to create an OpenURL link. However, in
+# our testing, many records don't have this at all. **Note** Ask EBSCO support
+# to configure your profile so OpenURLs are ALWAYS included for all records, not
+# just records with no EBSCO fulltext, to ensure bento_search can get the
+# openurl. 
+#
+# As always, you can customize links and other_links with Item Decorators. 
+#
+# == Technical Notes and Difficulties
+#
+# This API is enormously difficult to work with. Also the response is very odd
+# to deal with and missing some key elements. We quite possibly got something
+# wrong or non-optimal in this implementation, but we did our best. 
+#
+# Auth issues may make this slow -- you need to spend a (not too speedy) HTTP
+# request making a session for every new end-user -- as we have no way to keep
+# track of end-users, we do it on every request in this implementation. 
+#
+# Responses don't include much metadata -- we don't actually have journal title,
+# volume, issue, etc.  We probably _could_ parse it out of the OpenURL that's
+# there depending on your profile configuration, but we're not right now.
+# Instead we're using the chunk of user-displayable citation/reference it does
+# give us (which is very difficult to parse into something usable already),
+# and a custom Decorator to display that instead of normalized citation
+# made from individual elements. 
+#
+# EBSCO says they plan to improve some of these issues in a September 2012 release. 
+#
+# Title and abstract data seems to be HTML with tags and character entities and
+# escaped special chars. We're trusting it and passing it on as html_safe. 
+#
+# Paging can only happen on even pages, with 'page' rather than 'start'. But
+# you can pass in 'start' to bento_search, it'll be converted to closest page. 
+#
+# == Authenticated Users
+#
+# EDS allows searches by unauthenticated users, but the results come back with 
+# weird blank hits. In such a case, the BentoSearch adapter will return
+# records with virtually no metadata, but a title e
+# (I18n at bento_search.eds.record_not_available ). Also no abstracts
+# are available from unauth search. 
+#
+# By default the engine will search as 'guest' unauth user. But config
+# 'auth' key to true to force all searches to auth (if you are protecting your
+# app) or pass :auth => true as param into #search method.  
+#
+# == EDS docs:
+# 
+# * Console App to demo requests: https://eds-api.ebscohost.com/Console   
+# * EDS Wiki: http://edswiki.ebscohost.com/EDS_API_Documentation
+# * You'll need to request an account to the EDS wiki, see: http://support.ebsco.com/knowledge_base/detail.php?id=5990
+# 
+class BentoSearch::EdsEngine
+  include BentoSearch::SearchEngine
+  
+  extend HTTPClientPatch::IncludeClient
+  include_http_client
+    
+  AuthHeader          = "x-authenticationToken"
+  SessionTokenHeader  = "x-sessionToken"
+
+  @@remembered_auth = nil
+  @@remembered_auth_lock = Mutex.new
+  # Class variable to save current known good auth
+  # uses a mutex to be threadsafe. sigh. 
+  def self.remembered_auth
+    @@remembered_auth_lock.synchronize do      
+      @@remembered_auth
+    end
+  end
+  # Set class variable with current known good auth. 
+  # uses a mutex to be threadsafe. 
+  def self.remembered_auth=(token)
+    @@remembered_auth_lock.synchronize do
+      @@remembered_auth = token
+    end
+  end
+  
+  # an object that includes some Rails helper modules for
+  # text handling. 
+  def helper
+    unless @helper
+      @helper = Object.new
+      @helper.extend ActionView::Helpers::TextHelper # for truncate
+      @helper.extend ActionView::Helpers::OutputSafetyHelper # for safe_join
+    end
+    return @helper
+  end
+  
+  
+  def self.required_configuration
+    %w{user_id password profile}
+  end
+  
+  # From config or args, args over-ride config
+  def authenticated_end_user?(args)    
+    config = configuration.auth ? true : false
+    arg = args[:auth]
+    if ! arg.nil?
+      arg ? true : false
+    elsif ! config.nil?
+      config ? true : false
+    else
+      false
+    end
+  end
+  
+  def construct_search_url(args)
+    query = "AND,"
+    if args[:search_field]
+      query += "#{args[:search_field]}:"
+    end
+    # Can't have any commas in query, it turns out, although
+    # this is not documented. 
+    query += args[:query].gsub("/\,/", "")
+    
+    url = "#{configuration.base_url}search?view=detailed&query=#{CGI.escape query}"
+    
+    url += "&searchmode=#{CGI.escape configuration.search_mode}"
+    
+    url += "&highlight=#{configuration.highlighting ? 'y' : 'n' }"
+    
+    if args[:per_page]
+      url += "&resultsperpage=#{args[:per_page]}"
+    end
+    if args[:page]
+      url += "&pagenumber=#{args[:page]}"
+    end
+    
+    if args[:sort]
+      if (defn = self.sort_definitions[args[:sort]]) &&
+           (value = defn[:implementation] )
+        url += "&sort=#{CGI.escape value}"
+      end
+    end
+    
+    
+    return url
+  end
+  
+  
+  
+  def search_implementation(args)
+    results = BentoSearch::Results.new
+    
+    end_user_auth = authenticated_end_user? args
+    
+    begin
+      with_session(end_user_auth) do |session_token|
+                        
+        url = construct_search_url(args)
+                
+        response = get_with_auth(url, session_token)
+        
+        results = BentoSearch::Results.new
+        
+        if (hits_node = at_xpath_text(response, "./SearchResponseMessageGet/SearchResult/Statistics/TotalHits"))          
+          results.total_items = hits_node.to_i
+        end
+        
+        response.xpath("./SearchResponseMessageGet/SearchResult/Data/Records/Record").each do |record_xml|
+          item = BentoSearch::ResultItem.new
+          
+          item.title = prepare_eds_payload( element_by_group(record_xml, "Ti"), true )
+          if item.title.nil? && ! end_user_auth
+            item.title = I18n.translate("bento_search.eds.record_not_available")
+          end
+          
+          item.abstract = prepare_eds_payload( element_by_group(record_xml, "Ab"), true )
+
+          # Believe it or not, the authors are encoded as an escaped
+          # XML-ish payload, that we need to parse again and get the
+          # actual authors out of. WTF. Thanks for handling fragments
+          # nokogiri. 
+          author_mess = element_by_group(record_xml, "Au")
+          author_xml = Nokogiri::XML::fragment(author_mess)
+          author_xml.xpath(".//searchLink").each do |author_node|
+            item.authors << BentoSearch::Author.new(:display => author_node.text)
+          end
+          
+          
+          # PLink is main inward facing EBSCO link, put it as
+          # main link. 
+          if direct_link = record_xml.at_xpath("./PLink")
+              item.link = direct_link.text
+          end
+          
+          # Other links may be found in CustomLinks, it seems like usually
+          # there will be at least one, hopefully the first one is the OpenURL?
+          record_xml.xpath("./CustomLinks/CustomLink").each do |custom_link|
+            item.other_links << BentoSearch::Link.new(
+              :url => custom_link.at_xpath("./Url").text,
+              :label => custom_link.at_xpath("./Name").text
+              )
+          end
+          
+          if (configuration.assume_first_custom_link_openurl &&
+            (first = record_xml.xpath "./CustomLinks/CustomLink" ) &&
+            (node = first.at_xpath "./Url" )
+          )
+          
+            openurl = node.text
+            
+            index = openurl.index('?')
+            item.openurl_kev_co = openurl.slice index..(openurl.length) if index            
+          end
+
+          # Format. 
+          item.format_str = at_xpath_text record_xml, "./Header/PubType"
+          # Can't find a list of possible PubTypes to see what's there to try
+          # and map to our internal controlled vocab. oh wells. 
+          
+          
+          
+          # We have a single blob of human-readable citation, that's also
+          # littered with XML-ish tags we need to deal with. We'll save
+          # it in a custom location, and use a custom Decorator to display
+          # it. Sorry it's way too hard for us to preserve <highlight>
+          # tags in this mess, they will be lost. Probably don't
+          # need highlighting in source anyhow. 
+          citation_mess = element_by_group(record_xml, "Src")
+          citation_txt = Nokogiri::XML::fragment(citation_mess).text
+          # But strip off some "count of references" often on the end
+          # which are confusing and useless. 
+          item.custom_data["citation_blob"] = citation_txt.gsub(/ref +\d+ +ref\.$/, '')           
+                              
+          item.extend CitationMessDecorator
+          
+          results << item
+        end        
+      end
+      
+      return results      
+    rescue EdsCommException => e
+      results.error ||= {}
+      results.error[:exception] = e
+      results.error[:http_status] = e.http_status
+      results.error[:http_body] = e.http_body
+      return results
+    end
+    
+  end
+  
+  # Difficult to get individual elements out of an EDS XML <Record>
+  # response, requires weird xpath, so we do it for you. 
+  # element_by_group(nokogiri_element, "Ti")
+  #
+  # Returns string or nil
+  def element_by_group(noko, group)
+    at_xpath_text(noko, "./Items/Item[child::Group[text()='#{group}']]/Data")
+  end
+  
+  # Wraps calls to the EDS api with CreateSession and EndSession requests
+  # to EDS. Will pass sessionID in yield from block.
+  #
+  # Second optional arg is whether this is an authenticated user, else
+  # guest access will be used. 
+  #
+  #     with_session(true) do |session_token|
+  #       # can make more requests using session_token,
+  #       # EndSession will be called for you at end of block. 
+  #     end
+  def with_session(auth = false, &block)
+    auth_token = self.class.remembered_auth    
+    if auth_token.nil?
+      auth_token = self.class.remembered_auth = get_auth_token
+    end
+    
+            
+    create_url = "#{configuration.base_url}createsession?profile=#{configuration.profile}&guest=#{auth ? 'n' : 'y'}"    
+    response_xml = get_with_auth(create_url)    
+    
+    session_token = nil
+    unless response_xml && (session_token = at_xpath_text(response_xml, "//SessionToken"))  
+      e = EdsCommException.new("Could not get SessionToken")      
+    end
+                
+    begin    
+      block.yield(session_token)
+    ensure      
+      if auth_token && session_token   
+        end_url = "#{configuration.base_url}endsession?sessiontoken=#{CGI.escape session_token}"
+        response_xml = get_with_auth(end_url)        
+      end
+    end
+    
+  end
+  
+  # if the xpath responds, return #text of it, else nil. 
+  def at_xpath_text(noko, xpath)
+    node = noko.at_xpath(xpath)
+    
+    if node.nil?
+      return node
+    else
+      return node.text
+    end
+  end
+  
+  # If EDS has put highlighting tags
+  # in a field, we need to HTML escape the literal values,
+  # while still using the highlighting tokens to put
+  # HTML tags around highlighted terms.
+  #
+  # Second param, if to assume EDS literals are safe HTML, as they
+  # seem to be. 
+  def prepare_eds_payload(str, html_safe = false)
+    return str if str.blank?
+    
+    unless configuration.highlighting
+      str = str.html_safe if html_safe  
+      return str
+    end
+        
+    parts = 
+    str.split(%r{(</?highlight>)}).collect do |substr|
+      case substr
+      when "<highlight>" then "<b class='bento_search_highlight'>".html_safe
+      when "</highlight>" then "</b>".html_safe
+      # Yes, EDS gives us HTML in the literals, we're choosing to trust it. 
+      else substr.html_safe
+      end
+    end
+    
+    
+    
+    
+    
+    # Crazy ass method to truncate without getting in the middle of our
+    # html tags. This is wacky hacky, yeah. 
+    if configuration.truncate_highlighted
+      remainingLength = configuration.truncate_highlighted
+      in_tag = false
+      elipses_added = false
+      
+      truncated_parts = []
+      parts.each do |substr|
+        if remainingLength <=0 && ! in_tag
+          truncated_parts << "..."
+          break
+        end
+        
+        if substr =~ /^<b.*\>$/
+          truncated_parts << substr
+          in_tag = true
+        elsif substr == "</b>"
+          truncated_parts << substr
+          in_tag = false
+        elsif ((remainingLength - substr.length) > 0) || in_tag
+          truncated_parts << substr
+        else
+          truncated_parts << helper.truncate(substr, :length => remainingLength, :separator  => ' ')
+          break
+        end
+        
+        remainingLength = remainingLength - substr.length
+      end
+      
+      parts = truncated_parts
+    end
+
+    
+    return helper.safe_join(parts, '')        
+  end
+  
+  # Give it a url pointing at EDS API.
+  # Second arg must be a session_token if EDS request requires one. 
+  # It will 
+  # * Make a GET request
+  # * with memo-ized auth token added to headers
+  # * for XML, with all namespaces removed!
+  # * Parse JSON into a hash and return hash
+  # * Try ONCE more to get if EBSCO says bad auth token
+  # * Raise an EdsCommException if can't auth after second try,
+  #   or other error message, or JSON can't be parsed. 
+  def get_with_auth(url, session_token = nil)
+    auth_token = self.class.remembered_auth
+    unless auth_token
+      auth_token = self.class.remembered_auth = get_auth_token
+    end
+    
+    response = nil
+    response_xml = nil
+    caught_exception = nil
+                
+    begin
+      headers = {AuthHeader => auth_token, 'Accept' => 'application/xml'}
+      headers[SessionTokenHeader] = session_token if session_token
+      
+      s_time = Time.now
+      response = http_client.get(url, nil, headers)
+      Rails.logger.debug("EDS timing GET: #{Time.now - s_time}:#{url}")
+      
+      response_xml = Nokogiri::XML(response.body)
+      response_xml.remove_namespaces!
+      
+      if (at_xpath_text(response_xml, "//ErrorNumber") == "104") || (at_xpath_text(response_xml, "//ErrorDescription") == "Auth Token Invalid")
+        # bad auth, try again just ONCE
+        Rails.logger.debug("EDS auth failed, getting auth again")
+        
+        headers[AuthHeader] = self.class.remembered_auth = get_auth_token
+        response = http_client.get(url, nil, headers)
+        response_xml = Nokogiri::XML(response.body)
+        response_xml.remove_namespaces!        
+      end            
+    rescue TimeoutError, HTTPClient::ConfigurationError, HTTPClient::BadResponseError, Nokogiri::SyntaxError => e
+      caught_exception = e
+    end
+    
+    if response.nil? || response_xml.nil? || caught_exception ||  (! HTTP::Status.successful? response.status)
+      exception = EdsCommException.new("Error fetching URL: #{caught_exception.message if caught_exception} : #{url}")
+      if response
+        exception.http_body = response.body
+        exception.http_status = response.status
+      end
+      raise exception
+    end
+        
+    return response_xml
+  end
+  
+  
+  # Has to make an HTTP request to get EBSCO's auth token. 
+  # returns the auth token. We aren't bothering to keep
+  # track of the expiration ourselves, can't neccesarily trust
+  # it anyway. 
+  #
+  # Raises an EdsCommException on error. 
+  def get_auth_token    
+    # Can't send params as form-encoded, actually need to send a JSON or XML
+    # body, argh. 
+    
+    body = <<-EOS
+      {
+        "UserId":"#{configuration.user_id}",
+        "Password":"#{configuration.password}"
+      }
+    EOS
+    
+    s_time = Time.now
+    response = http_client.post(configuration.auth_url, body, {'Accept' => "application/json", "Content-type" => "application/json"})
+    Rails.logger.debug("EDS timing AUTH: #{Time.now - s_time}s")    
+    
+    unless HTTP::Status.successful? response.status
+      raise EdsCommException.new("Could not get auth", response.status, response.body)
+    end
+        
+    response_hash = nil
+    begin
+      response_hash = MultiJson.load response.body
+    rescue MultiJson::DecodeError
+    end
+  
+    unless response_hash.kind_of?(Hash) && response_hash.has_key?("AuthToken")
+      raise EdsCommException.new("AuthToken not found in auth response", response.status, response.body)
+    end
+    
+    return response_hash["AuthToken"]        
+  end
+  
+  def self.default_configuration
+    {
+      :auth_url => 'https://eds-api.ebscohost.com/authservice/rest/uidauth',
+      :base_url => "http://eds-api.ebscohost.com/edsapi/rest/",
+      :highlighting => true,
+      :truncate_highlighted => 280,
+      :assume_first_custom_link_openurl => true,
+      :search_mode => 'all' # any | bool | all | smart ; http://support.epnet.com/knowledge_base/detail.php?topic=996&id=1288&page=1
+    }
+  end
+  
+  def sort_definitions       
+    {        
+      "date_desc"     => {:implementation => "date"},
+      "relevance"     => {:implementation => "relevance" }
+      #       "date_asc"      => {:implementaiton => "date2"}
+    }
+  end
+  
+  def search_field_definitions
+    {
+      "TX" => {:semantic => :all},
+      "AU" => {:semantic => :author},
+      "TI" => {:semantic => :title},
+      "SU" => {:semantic => :subject},
+      "SO" => {}, # source, journal name
+      "AB" => {}, # abstract
+      "IS" => {:semantic => :issn},
+      "IB" => {:semantic => :isbn},
+    }
+  end
+  
+  # an exception talking to EDS api. 
+  # there's a short reason in #message, but also
+  # possibly an http_status and http_body copied
+  # from error EDS response. 
+  class EdsCommException < Exception
+    attr_accessor :http_status, :http_body
+    def initialize(message, status = nil, body = nil)
+      super(message)
+      self.http_status = status
+      self.http_body = body
+    end
+  end
+  
+  
+  # A built-in decorator alwasy applied, that over-rides
+  # the ResultItem#published_in display method to use our mess blob
+  # of human readable citation, since we don't have individual elements
+  # to create it from in a normalized way. 
+  module CitationMessDecorator
+    def published_in
+      custom_data["citation_blob"]
+    end
+  end
+  
+end