umlaut 3.2.0 → 3.3.0

data/README.md

@@ -24,7 +24,7 @@ advertise/recommend to it's users.
 
 Umlaut strives to supply links that take the user in as few clicks as possible to the service listed, without ever listing 'blind links' that you first have to click on to find out whether they are available. Umlaut pre-checks things when neccesary to only list services, with any needed contextual info, such that the user knows what they get when they click on it. Save the time of the user.
 
-[What do you mean by all this?](https://github.com/team-umlaut/umlaut/wiki/What-is-Umlaut-anyway)
+[What do you mean by all this?](https://github.com/team-umlaut/umlaut/wiki/What-is-Umlaut-anyway%3F)
 
 Umlaut is distributed as a ruby Rails engine gem. It's a very heavyweight engine,
 the point of distro'ing as a gem is to make it easy to keep local 

data/app/assets/stylesheets/umlaut/_resolve.scss

@@ -141,7 +141,7 @@
   @extend %indented-subitem;
 }
 
-.response_edition_warning, .response_source, .response_coverage_statement, .response_notes {
+.edition_warning, .response_source, .response_coverage_statement, .response_notes {
   @extend %indented-subitem;
   @extend .muted;
 }

data/app/controllers/feedback_controller.rb

@@ -0,0 +1,46 @@
+class FeedbackController < UmlautController
+  def new
+    contact_email_lookup(params[:contact_id])
+    # default render
+  end
+
+  def create    
+    contact_config = contact_email_lookup(params[:contact_id])
+    to_address = contact_config[:email_address]
+
+    options = params.slice(:name, :email, :feedback)
+    if params[:request_id] && umlaut_request = Request.find_by_id(params[:request_id])
+      options = options.merge(
+        :umlaut_request => umlaut_request
+      )
+    end
+
+    FeedbackMailer.feedback(request.host_with_port, to_address, options).deliver
+
+    flash[:alert_success] = "Thanks, your message has been sent."
+
+    if umlaut_request
+      redirect_to :controller => "resolve", :action => :index, "umlaut.request_id" => umlaut_request.id
+    else
+      redirect_to root_url
+    end
+  end
+
+  protected
+  def contact_email_lookup(contact_id)
+    unless contact_id
+      raise NoFeedbackEmailFoundException.new("Missing a contact_id, needed to look up feedback destination email.")
+    end
+    contact_config = umlaut_config.feedback && umlaut_config.feedback.contacts && umlaut_config.feedback.contacts[contact_id]
+
+    unless contact_config && contact_config[:email_address]
+      raise NoFeedbackEmailFoundException.new("Could not find feedback destination email for contact_id: `#{contact_id}`")
+    end
+
+    return contact_config
+  end
+
+  class NoFeedbackEmailFoundException < ArgumentError
+  end
+
+end

data/app/controllers/resource_controller.rb

@@ -7,6 +7,11 @@
 # an open proxy with the security problems that would cause.
 class ResourceController < UmlautController
   require 'open-uri'
+  require 'timeout'
+
+  # seconds to wait for thing were proxying. Yeah, fairly big num seems neccesary
+  # for Amazon at least.
+  HttpTimeout = 4
 
   # We really ought to _stream_ the remote response to our client, but I
   # couldn't get that to work how I wanted in Rails2. Even using
@@ -25,7 +30,11 @@ class ResourceController < UmlautController
     end
 
     proxied_headers = proxy_headers( request, uri.host )
-    remote_response = open(uri, 'rb', proxied_headers)
+
+    # open-uri :read_timeout is not behaving reliably, resort to Timeout.timeout
+    # should we just be using raw Net::Http and give up on open-uri? remember
+    # to update timeout exceptions in rescue below if you change.
+    remote_response = Timeout.timeout(HttpTimeout) {  open(uri, 'rb', proxied_headers) }
 
     # copy certain headers to our proxied response
     ["Content-Type", "Cache-Control", "Expires", "Content-Length", "Last-Modified", "Etag", "Date"].each do |key|
@@ -33,10 +42,19 @@ class ResourceController < UmlautController
       # rack doens't like it if we set a nil value here.
       response.headers[key] = value unless value.blank?
     end
+
     response.headers["X-Original-Url"] = url_str
 
     # And send the actual result out
     render(:text => remote_response.read)
+  rescue Timeout::Error, Errno::ECONNREFUSED, Errno::ETIMEDOUT => e
+    msg                = "#{e.inspect}: waiting for image proxy from `#{url_str}`; timeout is currently set to #{HttpTimeout}s; returning broken image"
+    relevant_backtrace = e.backtrace.find_all {|line| line =~ /umlaut/}
+
+    logger.warn("#{msg}\n  #{relevant_backtrace.join("\n  ")}")
+
+    response.headers['X-Original-Url'] = url_str
+    render :text => msg, :status => 504
   end
 
   protected

data/app/controllers/search_methods/sfx4.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require 'nokogiri'
 module SearchMethods
   module Sfx4
@@ -56,6 +57,7 @@ module SearchMethods
           # than 2 causes false negatives. Otherwise we use it to be consistent
           # with SFX. This reverse-engineering is full of pitfalls.
           query = terms.collect do |term|
+            term = replace_problem_tokens(term)
             "+" + connection.quote_string(term) + (term.length > 2 ? "*" : "")
           end.join(" ")
           "MATCH (TS.TITLE_SEARCH) AGAINST ('#{query}' IN BOOLEAN MODE)"
@@ -132,6 +134,19 @@ module SearchMethods
       return [context_objects, total_hits]
     end
 
+    # Query sanitising - remove bad characters from query
+    # (mimics SFX default behaviour - SFX stores them like this
+    # for sorting purposes)
+    # Note - this method is not complete, more substitutions may
+    # be necessary see the relevant Diacritics file in your SFX config
+    # directory for more details
+    def replace_problem_tokens(term)
+      term.gsub!('æ', 'a1')
+      term.gsub!('å', 'a2')
+
+      term
+    end
+
     # Used for clicks on A, B, C, 0-9, etc. 
     def find_by_group
       connection = sfx4_db_connection

data/app/controllers/umlaut_controller.rb

@@ -111,6 +111,12 @@ class UmlautController < ApplicationController
 
       end
 
+      # You can use Umlaut's built-in feedback form if you want
+      # See https://github.com/team-umlaut/umlaut/wiki/Umlaut's-Feedback-Form
+      # feedback do
+      #    main_library {:email_address => "dummy@example.org", :label => "Main Library"}
+      # end
+
       # Advanced topic, you can declaratively configure
       # what sections of the resolve page are output where
       # and how using resolve_sections and add_resolve_sections!

data/app/helpers/emailer_helper.rb

@@ -8,8 +8,9 @@ module EmailerHelper
     options[:include_labels] ||= false
     rv =""
     cite = request.referent.to_citation
-    title = truncate(cite[:title].strip, :length => 70,  :seperator => '...')
-    rv << (cite[:title_label].strip + ": ")if options[:include_labels]
+    title = truncate(cite[:title].strip, :length => 70,  :separator => ' ')
+
+    rv << (cite[:title_label].strip + ": ")if options[:include_labels] && cite[:title_label]
     rv << title
     rv << "\n"
     if cite[:author]
@@ -18,7 +19,7 @@ module EmailerHelper
       rv << "\n"
     end
     if cite[:subtitle]
-      rv << (cite[:subtitle_label].strip + ": ") if options[:include_labels]
+      rv << (cite[:subtitle_label].strip + ": ") if options[:include_labels] && cite[:subtitle_label]
       rv << cite[:subtitle].strip
       rv << "\n"
     end
@@ -33,4 +34,17 @@ module EmailerHelper
     end
     return rv
   end
+
+  def citation_identifiers(request, options = {})
+    citation = request.referent.to_citation
+    str = ""
+
+    str << "ISSN: #{citation[:issn]}\n" if citation[:issn]
+    str << "ISBN: #{citation[:isbn]}\n" if citation[:isbn]
+    citation[:identifiers].each do |identifier|
+      str << "#{identifier}\n"
+    end
+
+    return str
+  end
 end

data/app/mailers/feedback_mailer.rb

@@ -0,0 +1,25 @@
+class FeedbackMailer < ActionMailer::Base
+  add_template_helper(EmailerHelper)
+
+  default from: UmlautController.umlaut_config.from_email_addr
+
+  # feedback("findit.library.school.edu", "librarian@university.edu",:name => "Joe", :email => "joe@gmail.com", :feedback => "Whatever", :umlaut_request => urequest)
+  #   * umlaut_request is optional     
+  def feedback(host, to_address, options = {})
+    @host = host
+    @umlaut_request = options[:umlaut_request]
+    @name     = options[:name]
+    @email    = options[:email]
+    @feedback = options[:feedback]
+
+    # Force permalink creation if we don't have one already
+    if @umlaut_request && @umlaut_request.referent.permalinks.empty?
+      permalink = Permalink.new_with_values!(@umlaut_request.referent, @umlaut_request.referrer_id)            
+      @umlaut_request.referent.permalinks << permalink
+      @umlaut_request.save!
+    end
+
+    mail(:to => to_address, :subject => "#{UmlautController.umlaut_config.app_name} Feedback: #{options[:name]}", :reply_to => @email)
+  end
+
+end

data/app/mixin_logic/metadata_helper.rb

@@ -301,6 +301,43 @@ module MetadataHelper
     return nil
   end
 
+  def get_month(rft)
+    if rft.metadata['date'] =~ /\d\d\d\d\-(\d\d?)/
+      return $1
+    elsif rft.metadata['month']
+      # some link generators use an illegal 'month' parameter
+      return rft.metadata['month']
+    else
+      return nil
+    end
+  end
+
+  # uses `spage` or tries to parse `pages`
+  def get_spage(rft)
+    if rft.metadata['spage'].present?
+      return rft.metadata['spage']
+    elsif rft.metadata['pages'] =~ /\A *(.*?) *\-.*\Z/
+      return $1
+    elsif rft.metadata['pages'].present?
+      return rft.metadata['pages']
+    else
+      return nil
+    end
+  end
+
+  # uses `epage` or tries to parse `pages`
+  def get_epage(rft)
+    if rft.metadata['epage'].present?
+      return rft.metadata['epage']
+    elsif rft.metadata['pages'] =~ /\A.*\- *(.*) *\Z/
+      return $1
+    elsif rft.metadata['pages'].present?
+      return rft.metadata['pages']
+    else
+      return nil
+    end
+  end
+
   # Look at weird bad OpenURLs, use heuristics to see if the 'title' probably
   # represents a journal rather than a book. A guess at best, based on the bad
   # data we've seen, sigh. 

data/app/models/referent.rb

@@ -280,6 +280,8 @@ class Referent < ActiveRecord::Base
   end
 
   # Creates a hash for use in View code to display a citation
+  #
+  # Crazy if/else tree logic, should be refactored, needs more tests first prob. 
   def to_citation
     citation = {}
     # call self.metadata once and use the array for efficiency, don't
@@ -290,7 +292,7 @@ class Referent < ActiveRecord::Base
       citation[:title] = my_metadata['atitle']
       citation[:title_label], citation[:container_label] = 
         case my_metadata['genre']
-          when /article|journal|issue/ then ['Article Title', 'journal']
+          when /article|journal|issue/ then ['Article Title', 'Journal']
           when /bookitem|book/ then ['Chapter/Part Title', 'book']
           when /proceeding|conference/ then ['Proceeding Title', 'conference']
           when 'report' then ['Report Title','report']    
@@ -298,7 +300,7 @@ class Referent < ActiveRecord::Base
           if self.format == 'book'
             ['Chapter/Part Title', 'book']
           elsif self.format == 'journal'
-            ['Article Title', 'journal']
+            ['Article Title', 'Journal']
           else # default fall through, use much what SFX uses. 
             ['Title', '']
           end
@@ -312,11 +314,11 @@ class Referent < ActiveRecord::Base
       end
     else      
       citation[:title_label] = case my_metadata["genre"]
-        when /article|journal|issue/ then 'journal'
-        when /bookitem|book/ then 'book'
-        when /proceeding|conference/ then 'conference'
-        when 'report' then 'report'
-        else ''
+        when /article|journal|issue/i then 'Journal'
+        when /bookitem|book/i then 'Book'
+        when /proceeding|conference/i then 'Conference'
+        when 'report' then 'Report'
+        else nil
       end
       ['title','btitle','jtitle'].each do | t_type |
         if ! my_metadata[t_type].blank?
@@ -326,7 +328,7 @@ class Referent < ActiveRecord::Base
       end      
     end
     # add publisher for books
-    if (my_metadata['genre'] == 'book')
+    if (my_metadata['genre'] =~ /book/i)
       citation[:pub] = my_metadata['pub'] unless my_metadata['pub'].blank?
     end
 
@@ -336,21 +338,22 @@ class Referent < ActiveRecord::Base
     ['volume','issue','date'].each do | key |
       citation[key.to_sym] = my_metadata[key]
     end
-    if ! my_metadata["au"].blank?
-      citation[:author] = my_metadata["au"]
+
+    if my_metadata["au"].present?
+      citation[:author] = my_metadata["au"].strip
     elsif my_metadata["aulast"]
-      citation[:author] = my_metadata["aulast"]
-      if ! my_metadata["aufirst"].blank?
-        citation[:author] += ',	'+my_metadata["aufirst"]
+      citation[:author] = my_metadata["aulast"].strip
+      if my_metadata["aufirst"].present?
+        citation[:author] += ', '+my_metadata["aufirst"].strip
       else
-        if ! my_metadata["auinit"].blank?
-          citation[:author] += ',	'+my_metadata["auinit"]
+        if my_metadata["auinit"].present?
+          citation[:author] += ', '+my_metadata["auinit"].strip
         else
-          if ! my_metadata["auinit1"].blank?
-            citation[:author] += ',	'+my_metadata["auinit1"]
+          if my_metadata["auinit1"].present?
+            citation[:author] += ', '+my_metadata["auinit1"].strip
           end
-          if ! my_metadata["auinitm"].blank?
-            citation[:author] += my_metadata["auinitm"]
+          if my_metadata["auinitm"].present?
+            citation[:author] += my_metadata["auinitm"].strip
           end
         end
       end

data/app/models/request.rb

@@ -16,6 +16,8 @@ class Request < ActiveRecord::Base
   # responses show up first
   has_many :service_responses, :order => 'id ASC' 
 
+  has_many :clickthroughs
+
   belongs_to :referent, :include => :referent_values
   # holds a hash representing submitted http params
   serialize :http_env
@@ -35,6 +37,7 @@ class Request < ActiveRecord::Base
     
     # Create a context object from our http params
     context_object = OpenURL::ContextObject.new_from_form_vars( co_params )
+
     # Sometimes umlaut puts in a 'umlaut.request_id' parameter.
     # first look by that, if we have it, for an existing request.  
     request_id = params['umlaut.request_id']

data/app/referent_filters/dissertation_catch.rb

@@ -33,7 +33,6 @@ class DissertationCatch < ReferentFilter
     issn = get_identifier(:urn, "issn", referent)
     return unless issn
 
-    
     # normalize removing hyphen
     issn.gsub!('-', '')
     
@@ -66,6 +65,7 @@ class DissertationCatch < ReferentFilter
       referent.remove_value("issue_start")
       referent.remove_value("spage")
       referent.remove_value("epage")
+      referent.remove_value("pages")
     end
 
   end

data/app/service_adaptors/all_books_dot_com.rb

@@ -0,0 +1,17 @@
+require 'isbn_link'
+
+# Blind (not pre-checked for hits) link to compare prices
+# at AllBookstores.com
+# -- because that site seemed good, includes shipping prices, has decent UX, 
+# and includes independent bookstores like Powell's and The Strand. 
+#
+# subclasses IsbnLink
+class AllBooksDotCom < IsbnLink
+  def initialize(config)    
+    super(config)
+
+    @display_text   ||= "Compare online prices"
+    @display_name   ||= "AllBookstores.com"
+    @link_template  ||= "http://www.allbookstores.com/book/compare/%s"
+  end
+end

data/app/service_adaptors/illiad.rb

@@ -0,0 +1,161 @@
+require 'cgi'
+require 'openurl'
+
+# Just creates an OpenURL link out, corresponding to the current OpenURL. 
+# But tweaked in ways to try and work out for sending to ILLiad. 
+#
+# If you use SFX, you may want to just use SFX's built-in ILLiad targets, which
+# will generally be picked up by the SFX Umlaut service. 
+#
+# But if you don't or if you are unhappy with what SFX is doing, you could turn
+# off ILLiad target(s) in SFX (or configure Umlaut SFX adapter to ignore them), 
+# and use this instead. 
+#
+# # Pre-empting
+#
+# You may want to show ILLiad links only if there is no fulltext, or only if there
+# is no fulltext from a certain service. You can use Umlaut's standard service
+# pre-emption configuration for that. 
+#
+# Do not produce ILLiad links if there are ANY fulltext links already produced
+# in the request. In umlaut_services.yml:
+#
+#     illiad:
+#       type: Illiad
+#       priority: 4
+#       preempted_by:
+#         existing_type: fulltext
+#
+# Or, preempt ILLiad links only if there are fulltext links created by SFX
+# specifically (assume "SFX" is the id of your sfx service in umlaut_services.yml)
+#
+#     illiad:
+#       type: Illiad
+#       priority: 4
+#       preempted_by:
+#         existing_service: SFX
+#         existing_type: fulltext
+#
+# Pre-emption can only take account of services already generated before ILLiad
+# service is triggered, so you'd want to make sure to give ILLiad a priority greater
+# than the services you want to potentially preempt it. 
+#
+# # Config parameters
+# ## Required
+# 
+# * base_url: Illiad base url, such as `http://ill.university.edu/site/illiad.dll/OpenURL`. It should probably end in '/illiad.dll/OpenURL'
+#
+# ## Optional
+# 
+# * display_name: Default "Place ILL Request"
+# * sid_suffix: Default " (via Umlaut)", appended to existing sid before sending to ILLiad. 
+# * notes: Some additional notes to display under the link. 
+class Illiad < Service
+  include MetadataHelper
+
+  required_config_params :base_url
+
+  def initialize(config)
+    @service_type = "document_delivery"
+    @display_name = "Place ILL Request"
+    @sid_suffix   = " (via Umlaut)"
+
+    super(config)    
+  end
+
+
+  def service_types_generated
+    [ServiceTypeValue[@service_type.to_sym]]
+  end
+
+  def handle(request)   
+    target_url = @base_url + "?" + illiad_query_parameters(request).to_query
+
+    request.add_service_response(
+      :service            =>self, 
+      :display_text       => @display_name,
+      :url                => target_url,
+      :notes              => @notes, 
+      :service_type_value => @service_type.to_sym
+    )
+
+    return request.dispatched(self, true)
+  end
+
+  
+  def illiad_query_parameters(request)
+    metadata = request.referent.metadata
+
+    qp = {}
+
+    qp['genre']     = metadata['genre']
+
+    if metadata['aulast']
+      qp["aulast"]  = metadata['aulast']
+      qp["aufirst"] = [metadata['aufirst'], metadata["auinit"]].find {|a| a.present?}
+    else
+      qp["au"]      = metadata["au"]
+    end
+
+    qp['volume']    = metadata['volume']
+    qp['issue']     = metadata['issue']
+
+    qp['spage']     = get_spage(request.referent)
+    qp['epage']     = get_epage(request.referent)
+
+    qp['issn']      = get_issn(request.referent)
+    qp['isbn']      = get_isbn(request.referent)
+    qp['pmid']      = get_pmid(request.referent)
+
+    qp['stitle']    = metadata['stitle']
+
+    qp['sid']       = sid_for_illiad(request)
+
+    qp['year']      = get_year(request.referent)
+    qp['month']     = get_month(request.referent)
+
+    qp['atitle']    = metadata['atitle']
+
+    # ILLiad always wants 'title', not the various title keys that exist in OpenURL
+    qp['title']     = [metadata['jtitle'], metadata['btitle'], metadata['title']].find {|a| a.present?}
+
+    # For some reason these go to ILLiad prefixed with rft.
+    qp['rft.pub']     = metadata['pub']
+    qp['rft.place']   = metadata['place']
+    qp['rft.edition'] = metadata['edition']
+
+    # ILLiad likes OCLCnum in `rfe_dat`
+    qp['rfe_dat']   = get_oclcnum(request.referent)
+
+
+    # Genre normalization. ILLiad pays a lot of attention to `&genre`, but
+    # doesn't use actual OpenURL rft_val_fmt
+    if request.referent.format == "dissertation"
+      qp['genre'] = 'dissertation'
+    elsif qp['isbn'].present? && qp['genre'] == 'book' && qp['atitle'] && (! qp['issn'].present?)
+      # actually a book chapter, not a book, fix it. 
+      qp['genre'] = 'bookitem'
+    elsif qp['issn'].present? && qp['atitle'].present?
+      # Otherwise, if there is an ISSN, we force genre to 'article', seems
+      # to work best.  
+      qp['genre'] = 'article'      
+    elsif qp['genre'] == 'unknown' && qp['atitle'].blank?
+      # WorldCat likes to send these, ILLiad is happier considering them 'book'
+      qp['genre'] = "book"
+    end
+
+    # trim empty ones please
+    qp.delete_if {|k, v| v.blank?}
+    
+    return qp
+  end
+
+  # Grab a source label out of `sid` or `rfr_id`, add on our suffix. 
+  def sid_for_illiad(request)    
+    sid = request.referrer_id || ""
+
+    sid = sid.gsub(%r{\Ainfo\:sid/}, '')
+
+    return "#{sid}#{@sid_suffix}"
+  end
+end

data/app/service_adaptors/isbn_db.rb

@@ -2,6 +2,11 @@
 # info for online sellers. There are potentially other services we could
 # make use of there in the future too.
 #
+# By default makes an API request in advance to check if book is available, and thus requires
+# an API key.  However, ISBNdb seems no longer as reliable as it once was there. 
+# You may not want to use ISBNdb at all, see AllBooksDotCom as an alternative
+# (although with no pre-check for hits.)
+#
 # jrochkind talked to the operators of isbndb and got a very high traffic limit
 # (instead of the default free 500 requests/day), for free. You could try the
 # same. 
@@ -22,7 +27,7 @@ class IsbnDb < Service
   end
 
   def initialize(config)
-    @timeout = 7
+    @timeout = 3
     @display_text = "Compare online prices"
     @display_name = "ISBNdb.com"
     

data/app/service_adaptors/isbn_link.rb

@@ -0,0 +1,57 @@
+require 'isbn'
+
+# A simple service to generate a blind link (NOT pre-checked for hits, just
+# blindly created from a template) out to a service based on ISBN. 
+#
+# May likely be sub-classed for specific services (see AllBooks.com),
+# which set default values. 
+#
+# * :link_template. => String where "%s" will be replaced with ISBN
+# * :display_name 
+# * :dispaly_text. Such as "Compare online prices
+# * :isbn_normalize. Default nil, set to :ten or :thirteen if you need to normalize
+#     ISBN before substituting in :link_template. 
+class IsbnLink < Service
+  include MetadataHelper
+      
+  def service_types_generated
+    return [ServiceTypeValue['highlighted_link']]
+  end
+
+  def initialize(config)    
+    @display_text   = "Compare online prices"
+    @isbn_normalize = nil
+    
+    super(config)
+  end
+
+  def handle(umlaut_request)
+    
+    isbn = get_isbn(umlaut_request.referent)
+    
+    # No isbn, nothing we can do. 
+    return umlaut_request.dispatched(self, true) if isbn.blank?
+
+    # invalid isbn? forget it. 
+    return umlaut_request.dispatched(self, true) unless ISBN.valid?(isbn)
+
+    if @isbn_normalize == :ten
+      isbn = ISBN.ten(isbn)    
+    elsif @isbn_normalize == :thirteen
+      isbn = ISBN.thirteen(isbn)
+    end
+    
+    # Add the link
+    link = @link_template.gsub("%s", isbn)
+    
+    umlaut_request.add_service_response(
+      :service=>self, 
+      :url=> link, 
+      :display_text=> @display_text,
+      :service_type_value => ServiceTypeValue[:highlighted_link]
+    )
+
+    return umlaut_request.dispatched(self, true)
+  end
+    
+end

data/app/service_adaptors/scopus.rb

@@ -1,4 +1,8 @@
 # Service adapter plug-in.
+#
+# NOTE: This is based on deprecated Scopus API's, Scopus will take them
+# away 31 December 2014. Please see scopus2.rb instead which uses
+# new Scopus API's. 
 # 
 # PURPOSE: Includes "cited by", "similar articles" and "more by these authors"
 # links from scopus. Also will throw in an abstract from Scopus if found.