umlaut 3.0.0alpha1

data/lib/service_adaptors/elsevier_cover.rb

@@ -0,0 +1,57 @@
+# Elsevier provides publically available and linkable sample cover images
+# for journals they publish. Thanks Elsevier! This service does nothing
+# more than take an ISSN and look for a match from Elsevier.
+class ElsevierCover < Service
+  require 'open-uri'
+  
+  def service_types_generated
+    return [ServiceTypeValue[:cover_image]]
+  end
+
+  def initialize(config)
+    #@base_url = "http://www1.elsevier.com/inca/covers/store/issn/"
+    @base_url = "http://www.extranet.elsevier.com/inca_covers_store/issn/"
+    
+    super(config)
+  end
+  
+  def handle(request)
+    issn = request.referent.issn
+
+    # We need an ISSN
+    return request.dispatched(self, true) unless issn 
+
+    # No hyphens please
+    issn = issn.gsub(/[^0-9X]/, '')
+    
+    check_url = @base_url + issn + '.gif'
+
+    # does it exist?
+    if ( url_resolves(check_url)   )
+       request.add_service_response(:service => self,
+                                    :service_type_value => ServiceTypeValue[:cover_image] ,
+                                    :url => check_url, 
+                                    :size => "medium" )
+    end   
+    
+    return request.dispatched(self, true)
+  end
+
+  def url_resolves(url)
+    uri_obj = URI.parse(url)
+    response = Net::HTTP.start(uri_obj.host, uri_obj.port) {|http|
+      http.head(uri_obj.request_uri)
+    }
+    if (response.kind_of?( Net::HTTPSuccess  ))
+      return true
+    elsif ( response.kind_of?(Net::HTTPNotFound))
+      return false
+    else
+      # unexpected condition, raise
+      response.value
+    end
+
+    
+  end
+  
+end

data/lib/service_adaptors/email_export.rb

@@ -0,0 +1,10 @@
+class EmailExport < AjaxExport
+
+  def initialize(config)
+    @display_text ||= "Email"
+    @form_controller ||= "export_email"
+    @form_action ||= "email"
+    super(config)
+  end
+
+end

data/lib/service_adaptors/ezproxy.rb

@@ -0,0 +1,171 @@
+#  By default, proxies a URL after checking the EZProxy API to see if
+#  it's proxy-able. But you can set the config param precheck_with_api
+#  to false, and then this will simply automatically proxy all links
+#  from umlaut responses.  That is useful if you have your EZProxy
+#  server set to automatically redirect non-proxyable URLs to the original
+#  non-proxied version, the API check may not be neccesary. 
+
+# Required parameters:
+#   proxy_server: hostname of EZProxy instance (no "http://", just hostname)
+#
+#   optional params:
+#   proxy_password (the ProxyURLPassword  parameter in ezproxy.cfg; must be set
+#                   to turn on proxy url api feature ).
+#   proxy_url_path: defaults to /proxy_url, the default ezproxy path to call api
+#   exclude_hosts: array of hosts to exclude from proxying _even if_ found in
+#                  ezproxy config. Each entry can be a string, in which
+#                  case it must match host portion of url exactly. Or it can 
+#                  be a regexp, which will be tested against entire url.
+#                  (supply a string inside // markers. eg '/regexp/' ).
+#
+#   This service is a link_out_filter service, it must be setup in your
+#   services.yml with "task: link_out_filter ".
+
+
+class Ezproxy < Service
+  required_config_params :proxy_server
+  
+  require 'rexml/document'
+  require 'uri'
+  require 'net/http'
+  require 'cgi'  
+
+  def initialize(config)
+    @precheck_with_api = true
+    @display_name = "EZProxy"
+    @proxy_login_path = "/login"
+    
+    super(config)
+    
+    
+    @proxy_url_path ||= "/proxy_url"
+    @proxy_url_path = "/" + @proxy_url_path unless @proxy_url_path[0,1] = '/'
+
+    @exclude ||= []
+  end
+
+  # This is meant to be called as task:link_out_filter, it doesn't have an
+  # implementation for handle, it implements link_out_filter() instead. 
+  def handle(request)
+     raise "Not implemented."
+  end
+
+  # Hook method called by Umlaut. 
+  # Returns a proxied url if it should be proxied, or nil if the url
+  # can not or does not need to be proxied. 
+  def link_out_filter(orig_url, service_response, other_args = {})
+    # remove trailing or leading whitespace from url, it makes it
+    # an illegal URL anyway, but maybe we can rescue it? Marc 856's
+    # sometimes have accidental trailing whitespace. 
+    orig_url = orig_url.strip
+    
+    # bad uri? Forget it.    
+    return nil unless valid_url?( orig_url )
+    
+    # If it's already proxied, leave it alone.
+    return nil if already_proxied(orig_url)
+
+    return nil if excluded?(orig_url)
+
+    new_url = nil
+    if @precheck_with_api
+      new_url =  check_proxy_urls( [orig_url] ).values[0]
+    else
+      new_url =  auto_proxy_url(orig_url)
+    end
+    
+    return new_url
+  end
+
+  def valid_url?(url)
+    begin
+      raise Exception.new("Empty url!") if url.blank?
+      URI.parse( url )
+      return true
+    rescue Exception => e
+      Rails.logger.error("Bad uri sent to ezproxy service. Can not parse. url: <#{url}>")
+      return false
+    end
+  end
+
+  # see @exclude config parameter. 
+  def excluded?(url)    
+    return false if @exclude.blank?
+    
+    @exclude.each do |entry|
+      if ((entry[0,1] == '/') && (entry[entry.length()-1 ,1 ] == '/'))
+        # regexp. Match against entire url. 
+        re = Regexp.new( entry )
+        return true if re =~ url
+      elsif (entry.kind_of? Regexp)
+        return true if entry =~ url
+      else
+        # ordinary string. Just match against host.
+        host = URI.parse(url).host
+        return true if host == entry
+      end      
+    end
+    # looped through them all, no match?
+    return false
+  end
+
+  # pass in a url, this just mindlessly sends it through your
+  # ezproxy instance. 
+  def auto_proxy_url(url)
+    return "http://" + @proxy_server + @proxy_login_path + "?qurl=" + CGI.escape(url)
+  end
+  
+  # Pass in an array of URLs. Will determine if they are proxyable by EZProxy.
+  # Returns a hash, where the key is the original URL, and the value is the
+  # proxied url---or nil if could not be proxied. 
+  def check_proxy_urls(urls)
+    url_doc = REXML::Document.new
+    doc_root = url_doc.add_element "proxy_url_request", {"password"=>@proxy_password}
+    urls_elem = doc_root.add_element "urls"
+    urls.each { | link |
+      url_elem = urls_elem.add_element "url"
+      url_elem.text = link
+    }
+    begin
+      resp = Net::HTTP.post_form(URI.parse('http://' + @proxy_server+@proxy_url_path), {"xml"=>url_doc.to_s})    
+      proxy_doc = REXML::Document.new resp.body
+    rescue Timeout::Error
+      Rails.logger.error "Timed out connecting to EZProxy"
+      return proxy_links
+    rescue Exception => e
+      Rails.logger.error "EZProxy error, NOT proxying URL + #{e}"
+    end
+  
+    return_hash = {}
+    REXML::XPath.each(proxy_doc, "/proxy_url_response/proxy_urls/url") { | u |
+      unless (u && u.get_text) # if u is empty... weird, but skip it.
+        Rails.logger.error "EZProxy response seems to be missing some pieces.\n   Urls requested: #{urls.join(',')}\n   EZProxy api request xml: #{url_doc.to_s}\n   EZProxy response: #{proxy_doc.to_s}"
+      end
+      orig_url = u.get_text.value
+      return_hash[orig_url] = nil
+    
+      if u.attributes["proxy"] == "true"
+        proxied_url = u.attributes["scheme"]+"://"+u.attributes["hostname"]+":"+u.attributes["port"]+u.attributes["login_path"]
+        if u.attributes["encode"] == "true"
+          proxied_url += CGI::escape(u.get_text.value)
+        else
+          proxied_url += u.get_text.value
+        end
+
+        return_hash[orig_url] = proxied_url
+                
+      end
+    }    
+    return return_hash
+  end
+
+  # pass in url as a string. Return true if the
+  # url is already pointing to the proxy server
+  # configured. 
+  def already_proxied(url)
+    uri_obj = URI.parse(url)
+
+    return uri_obj.host == @proxy_server && uri_obj.path == @proxy_login_path
+  end
+
+end

data/lib/service_adaptors/google_book_search.rb

@@ -0,0 +1,442 @@
+# Service that searches Google Book Search to determine viewability.
+# It searches by ISBN, OCLCNUM and/or LCCN. 
+#
+# Uses Google Books API, http://code.google.com/apis/books/docs/v1/getting_started.html 
+# http://code.google.com/apis/books/docs/v1/using.html
+# 
+# If a full view is available it returns a fulltext service response.
+# If partial view is available, return as "limited experts". 
+# If no view at all, still includes a link in highlighted_links, to pay
+#   lip service to google branding requirements. 
+# Unfortunately there is no way tell which of the noview 
+# books provide search, although some do -- search is advertised if full or 
+# partial view is available. 
+# 
+# If a thumbnail_url is returned in the responses, a cover image is displayed.
+#
+# = Google API Key
+# 
+# Setting an api key in :api_key STRONGLY recommended, or you'll
+# probably get rate limited (not clear what the limit is with no api
+# key supplied). You may have to ask for higher rate limit for your api
+# key than the default 1000/day, which you can do through the google 
+# api console:
+# https://code.google.com/apis/console
+#
+# I requested 50k with this message, and was quickly approved with no questions
+# "Services for academic library (Johns Hopkins Libraries) web applications  to match Google Books availability to items presented by our catalog, OpenURL link resolver,  and other software. "
+#
+# Recommend setting your 'per user limit' to something crazy high, as well
+# as requesting more quota. 
+class GoogleBookSearch < Service
+  require 'multi_json'
+  
+  
+  # Identifiers used in API response to indicate viewability level
+  ViewFullValue = 'ALL_PAGES'
+  ViewPartialValue = 'PARTIAL'
+  # None might also be 'snippet', but Google doesn't want to distinguish
+  ViewNoneValue = 'NO_PAGES'
+  ViewUnknownValue = 'UNKNOWN'
+      
+  
+  
+  include MetadataHelper
+  include UmlautHttp
+  
+  # required params
+  
+  # attr_reader is important for tests
+  attr_reader :url, :display_name, :num_full_views 
+  
+  def service_types_generated
+    types= [
+      ServiceTypeValue[:fulltext], 
+      ServiceTypeValue[:cover_image],
+      ServiceTypeValue[:highlighted_link],
+      ServiceTypeValue[:search_inside],
+      ServiceTypeValue[:excerpts]]
+    types.push(ServiceTypeValue[:referent_enhance]) if @referent_enhance
+    return types
+  end
+  
+  def initialize(config)    
+    @url = 'https://www.googleapis.com/books/v1/volumes?q='
+    
+    @display_name = 'Google Books'
+    # number of full views to show
+    @num_full_views = 1
+    # default on, to enhance our metadata with stuff from google
+    @referent_enhance = true
+    # google api key strongly recommended, otherwise you'll
+    # probably get rate limited. 
+    @api_key = nil
+    
+    @credits = {
+      "Google Books" => "http://books.google.com/"
+    }
+    
+    super(config)
+  end
+  
+  def handle(request)
+
+    bibkeys = get_bibkeys(request.referent)
+    return request.dispatched(self, true) if bibkeys.nil?
+    data = do_query(bibkeys, request)
+    
+    
+    if data.blank? || data["error"]
+      # fail fatal
+      return request.dispatched(self, false)
+    end
+    
+    # 0 hits, return. 
+    return request.dispatched(self, true) if data["totalItems"] == 0
+    
+    enhance_referent(request, data) if @referent_enhance
+    
+    #return full views first
+    full_views_shown = create_fulltext_service_response(request, data)
+    
+    # Add search_inside link if appropriate
+    add_search_inside(request, data)
+    
+    # only if no full view is shown, add links for partial view or noview
+    unless full_views_shown
+      do_web_links(request, data)
+    end
+    
+    thumbnail_url = find_thumbnail_url(data)
+    if thumbnail_url
+      add_cover_image(request, thumbnail_url)    
+    end
+
+    return request.dispatched(self, true)
+  end
+
+  # Take the FIRST hit from google, and use it's values to enhance
+  # our metadata. Will NOT overwrite existing data. 
+  def enhance_referent(request, data)
+    
+    entry = data["items"].first
+    
+
+    if (volumeInfo = entry["volumeInfo"])
+      
+      title = volumeInfo["title"]
+      title += ": #{volumeInfo["subtitle"]}" if (title && volumeInfo["subtitle"])
+      
+      element_enhance(request, "title", title)
+      element_enhance(request, "au", volumeInfo["authors"].first) if volumeInfo["authors"]
+      element_enhance(request, "pub", volumeInfo["publisher"])
+      
+      element_enhance(request, "tpages", volumeInfo["pageCount"])
+      
+      if (date = volumeInfo["publishedDate"] && date =~ /^(\d\d\d\d)/)
+        element_enhance(request, "date", $1)
+      end
+      
+      # LCCN is only rarely included, but is sometimes, eg:
+      # "industryIdentifiers"=>[{"type"=>"OTHER", "identifier"=>"LCCN:72627172"}],          
+      # Also "LCCN:76630875"
+      #
+      # And sometimes OCLC number like:
+      # "industryIdentifiers"=>[{"type"=>"OTHER", "identifier"=>"OCLC:12345678"}],
+      #        
+      (volumeInfo["industryIdentifiers"] || []).each do |hash|
+        
+        if hash["type"] == "ISBN_13"
+          element_enhance(request, "isbn", hash["identifier"])
+          
+        elsif hash["type"] == "OTHER" && hash["identifier"].starts_with?("LCCN:")
+          lccn = normalize_lccn(  hash["identifier"].slice(5, hash["identifier"].length)  )
+          request.referent.add_identifier("info:lccn/#{lccn}")
+          
+        elsif hash["type"] == "OTHER" && hash["identifier"].starts_with?("OCLC:")
+          oclcnum = normalize_lccn(  hash["identifier"].slice(5, hash["identifier"].length)  )
+          request.referent.add_identifier("info:oclcnum/#{oclcnum}")
+        end
+      
+      end              
+    end            
+  end
+
+  # Will not over-write existing referent values. 
+  def element_enhance(request, rft_key, value)
+    if (value)
+      request.referent.enhance_referent(rft_key, value.to_s, true, false, :overwrite => false)
+    end
+  end
+
+  
+  # returns nil or escaped string of bibkeys
+  # to increase the chances of good hit, we send all available bibkeys 
+  # and later dedupe by id.
+  # FIXME Assumes we only have one of each kind of identifier.
+  def get_bibkeys(rft)
+    isbn = get_identifier(:urn, "isbn", rft)
+    oclcnum = get_identifier(:info, "oclcnum", rft)
+    lccn = get_lccn(rft)
+
+    # Google doesn't officially support oclc/lccn search, but does
+    # index as token with prefix smashed up right with identifier
+    # eg http://books.google.com/books/feeds/volumes?q=OCLC32012617
+    #
+    # Except turns out doing it as a phrase search is important! Or
+    # google's normalization/tokenization does odd things. 
+    keys = []
+    keys << ('isbn:' + isbn) if isbn
+    keys << ('"' + "OCLC" + oclcnum + '"') if oclcnum
+    # Only use LCCN if we've got nothing else, it returns many
+    # false positives. 
+    keys << ('"' + 'LCCN' + lccn + '"') if lccn && keys.length == 0
+    
+    return nil if keys.empty?
+    keys = CGI.escape( keys.join(' OR ') )
+    return keys
+  end
+  
+  def do_query(bibkeys, request)    
+    headers = build_headers(request)
+    link = @url + bibkeys
+    if @api_key
+      link += "&key=#{@api_key}"
+    end
+    
+    # Add on limit to only request books, not magazines. 
+    link += "&printType=books"
+
+    Rails.logger.debug("GoogleBookSearch requesting: #{link}")        
+    response = http_fetch(link, :headers => headers, :raise_on_http_error_code => false)        
+    data = MultiJson.decode(response.body)
+    
+    # If Google gives us an error cause it says it can't geo-locate, 
+    # remove the IP, log warning, and try again. 
+    
+    if (data["error"] && data["error"]["errors"] &&
+        data["error"]["errors"].find {|h| h["reason"] == "unknownLocation"} )
+      Rails.logger.warn("GoogleBookSearch: geo-locate error, retrying without X-Forwarded-For: '#{link}' headers: #{headers.inspect} #{response.inspect}\n    #{data.inspect}")
+      
+      response = http_fetch(link, :raise_on_http_error_code => false)        
+      data = MultiJson.decode(response.body)
+        
+    end
+    
+    
+    if (! response.kind_of?(Net::HTTPSuccess)) || data["error"]      
+      Rails.logger.error("GoogleBookSearch error: '#{link}' headers: #{headers.inspect} #{response.inspect}\n    #{data.inspect}")
+    end
+        
+    return data
+  end
+  
+  # We don't need to fake a proxy request anymore, but we still
+  # include X-Forwarded-For so google can return location-appropriate
+  # availability. If there's an existing X-Forwarded-For, we respect
+  # it and add on to it. 
+  def build_headers(request)
+    original_forwarded_for = nil
+    if (request.http_env && request.http_env['HTTP_X_FORWARDED_FOR'])
+      original_forwarded_for = request.http_env['HTTP_X_FORWARDED_FOR']                                  
+    end
+
+    # we used to prepare a comma seperated list in x-forwarded-for if
+    # we had multiple requests, as per the x-forwarded-for spec, but I
+    # think Google doesn't like it. 
+    
+    ip_address = (original_forwarded_for ?
+        original_forwarded_for  :
+        request.client_ip_addr.to_s)
+    
+    return {} if ip_address.blank?
+
+    # If we've got a comma-seperated list from an X-Forwarded-For, we
+    # can't send it on to google, google won't accept that, just take
+    # the first one in the list, which is actually the ultimate client
+    # IP. split returns the whole string if seperator isn't found, convenient.
+    ip_address = ip_address.split(",").first
+    
+    # If all we have is an internal/private IP from the internal network,
+    # do NOT send that to Google, or Google will give you a 503 error
+    # and refuse to process your request, as of 7 sep 2011. sigh.
+    # Also if it doesn't look like an IP at all, forget it, don't send it.     
+    if ((! ip_address =~ /^\d+\.\d+\.\d+\/\d$/) || 
+       ip_address.start_with?("10.") || 
+       ip_address.start_with?("172.16") || 
+       ip_address.start_with?("192.168"))
+       return {}
+    else    
+      return {'X-Forwarded-For' => ip_address }
+    end
+  end
+  
+  def find_entries(gbs_response, viewabilities)
+    unless (viewabilities.kind_of?(Array))
+      viewabilities = [viewabilities]
+    end
+
+    entries = gbs_response["items"].find_all do |entry|
+      viewability = entry["accessInfo"]["viewability"]
+      (viewability && viewabilities.include?(viewability))           
+    end
+
+    return entries
+  end
+  
+  
+  # We only create a fulltext service response if we have a full view.
+  # We create only as many full views as are specified in config.
+  def create_fulltext_service_response(request, data)
+    display_name = @display_name
+
+    full_views = find_entries(data, ViewFullValue)
+    return nil if full_views.empty?
+    
+    count = 0
+    full_views.each do |fv|
+      
+      uri = fv["volumeInfo"]["previewLink"]
+          
+      request.add_service_response(
+          :service => self, 
+          :display_text => display_name, 
+          :url => remove_query_context(uri),           
+          :service_type_value =>  :fulltext  
+      )
+      count += 1
+      break if count == @num_full_views
+    end   
+    return true
+  end
+
+  def add_search_inside(request, data)
+    # Just take the first one we find, if multiple
+    searchable_view = find_entries(data, [ViewFullValue, ViewPartialValue])[0]        
+    
+    if ( searchable_view )
+      url = searchable_view["volumeInfo"]["infoLink"]
+      
+      request.add_service_response( 
+        :service => self,
+        :display_text=>@display_name,
+        :url=> remove_query_context(url),
+        :service_type_value => :search_inside
+       )                  
+    end
+    
+  end
+  
+  # create highlighted_link service response for partial and noview
+  # Only show one web link. prefer a partial view over a noview.
+  # Some noviews have a snippet/search, but we have no way to tell. 
+  def do_web_links(request, data)
+
+    # some noview items will have a snippet view, but we have no way to tell
+    info_views = find_entries(data, ViewPartialValue)
+    viewability = ViewPartialValue
+    
+    if info_views.blank?
+      info_views = find_entries(data, ViewNoneValue)
+      viewability = ViewNoneValue  
+    end
+    
+    # Shouldn't ever get to this point, but just in case
+    return nil if info_views.blank?
+    
+    url = ''
+    iv = info_views.first
+    type = nil
+    if (viewability == ViewPartialValue && 
+        url = iv["volumeInfo"]["previewLink"])
+      display_text = @display_name
+      type = ServiceTypeValue[:excerpts]
+    else
+      url = url = iv["volumeInfo"]["infoLink"]
+      display_text = "Book Information"
+      type = ServiceTypeValue[:highlighted_link]
+    end
+    request.add_service_response( 
+        :service=>self,    
+        :url=> remove_query_context(url),
+        :display_text=>display_text,
+        :service_type_value => type    
+     )
+  end
+  
+
+  
+ 
+  # Not all responses have a thumbnail_url. We look for them and return the 1st.
+  def find_thumbnail_url(data)
+    entries = data["items"].collect do |entry|      
+      entry["volumeInfo"]["imageLinks"]["thumbnail"] if entry["volumeInfo"] && entry["volumeInfo"]["imageLinks"]      
+    end
+    
+    # removenill values
+    entries.compact!    
+    
+    # pick the first of the available thumbnails, or nil
+    return entries[0]
+  end
+  
+
+  def add_cover_image(request, url)
+    zoom_url = url.clone
+    
+    # if we're sent to a page other than the frontcover then strip out the
+    # page number and insert front cover
+    zoom_url.sub!(/&pg=.*?&/, '&printsec=frontcover&')
+    
+    # hack out the 'curl' if we can
+    zoom_url.sub!('&edge=curl', '')
+    
+    request.add_service_response(
+        :service=>self, 
+        :display_text => 'Cover Image',
+        :url => zoom_url, 
+        :size => "medium",
+        :service_type_value => :cover_image
+    )     
+  end
+  
+  # Google gives us URL to the book that contains a 'dq' param
+  # with the original query, which for us is an ISSN/LCCN/OCLCnum query,
+  # which we don't actually want to leave in there. 
+  def remove_query_context(url)
+    url.sub(/&dq=[^&]+/, '')    
+  end
+
+  # Catch url_for call for search_inside, because we're going to redirect
+  def response_url(service_response, submitted_params)
+    if ( ! (service_response.service_type_value.name == "search_inside" ))
+      return super(service_response, submitted_params)
+    else
+      # search inside!
+      base = service_response[:url]
+      query = CGI.escape(submitted_params["query"] || "")
+      # attempting to reverse engineer a bit to get 'snippet'
+      # style results instead of 'onepage' style results. 
+      # snippet seem more user friendly, and are what google's own
+      # interface seems to give you by default. but 'onepage' is the
+      # default from our deep link, but if we copy the JS hash data,
+      # it looks like we can get Google to 'snippet'.       
+      url = base + "&q=#{query}#v=snippet&q=#{query}&f=false"
+      return url
+    end
+  end
+  
+end
+
+# Important to quote search, see: "OCLC1246014"
+
+# Test WorldCat links
+# FIXME: This produces two 'noview' links because the ids don't match.
+#   This might be as good as we can do though, unless we want to only ever show
+#   one 'noview' link. Notice that the metadata does differ between the two.
+# http://localhost:3000/resolve?url_ver=Z39.88-2004&rfr_id=info%3Asid%2Fworldcat.org%3Aworldcat&rft_val_fmt=info%3Aofi%2Ffmt%3Akev%3Amtx%3Abook&req_dat=%3Csessionid%3E&rft_id=info%3Aoclcnum%2F34576818&rft_id=urn%3AISBN%3A9780195101386&rft_id=urn%3AISSN%3A&rft.aulast=Twain&rft.aufirst=Mark&rft.auinitm=&rft.btitle=The+prince+and+the+pauper&rft.atitle=&rft.date=1996&rft.tpages=&rft.isbn=9780195101386&rft.aucorp=&rft.place=New+York&rft.pub=Oxford+University+Press&rft.edition=&rft.series=&rft.genre=book&url_ver=Z39.88-2004
+#
+# Snippet view returns noview through the API
+# http://localhost:3000/resolve?rft.isbn=0155374656
+#
+# full view example, LCCN 07020699  ; OCLC: 1246014