umlaut 3.0.0alpha1

data/app/controllers/resource_controller.rb

@@ -0,0 +1,83 @@
+# We proxy cover images from foreign sources through Umlaut sometimes, in order
+# to switch their access from http to https to avoid browser warnings on an
+# https page.  This controller does that.
+#
+# It does NOT take URL in request parameters, but instead takes a response ID.
+# it will only proxy urls already stored in umlaut responses, so this is not
+# an open proxy with the security problems that would cause. 
+class ResourceController < UmlautController
+  require 'open-uri'
+
+  # We really ought to _stream_ the remote response to our client, but I
+  # couldn't get that to work how I wanted in Rails2. Even using
+  # render :text=>proc, problem is we needed to know the content-type before
+  # we read any of the response, which we didn't. This implementation holds
+  # the whole image in memory for a second while it delivers it, oh well.
+  # doesn't seem to effect speed much, even though it's not optimal. 
+  def proxy
+    svc_type = ServiceType.find(params[:id])      
+    url_str =  svc_type.view_data[:url]
+    uri = nil
+    begin
+      uri = URI(url_str)
+    rescue
+      raise Exception.new("resource#proxy can only handle String urls, not '#{url.inspect}'")
+    end
+
+    proxied_headers = proxy_headers( request, uri.host )
+    remote_response = open(uri, 'rb', proxied_headers)
+
+    # copy certain headers to our proxied response
+    ["content-type", "cache-control", "expires", "content-length", "last-modified", "etag", "date"].each do |key|
+      response.headers[key] = remote_response.meta[key]
+    end
+    response.headers["X-Original-Url"] = url_str
+    
+    # And send the actual result out
+    render(:text => remote_response.read)
+    
+  end
+
+  protected
+
+  # Generate headers as if we are a proxy server, to be more or less honest
+  # and to maybe keep Google et al from rate throttling us.
+  # This is kind of copy and paste of UmlautHttp#proxy_like_headers, but that
+  # method was written to require an UmlautRequest, which we don't have here.
+  # TODO should refactor to DRY.
+  # Argument here is a Rails Request
+  def proxy_headers( request, host  )
+    orig_env = request.headers  
+  
+    header = {}
+    
+  
+    header["User-Agent"] = orig_env['HTTP_USER_AGENT'] || 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008061015 Firefox/3.0'
+    header['Accept'] = orig_env['HTTP_ACCEPT'] || 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+    header['Accept-Language'] = orig_env['HTTP_ACCEPT_LANGUAGE'] || 'en-us,en;q=0.5'
+    header['Accept-Encoding'] = orig_env['HTTP_ACCEPT_ENCODING'] || ''
+    header["Accept-Charset"] = orig_env['HTTP_ACCEPT_CHARSET'] || 'UTF-8,*'
+
+    # Set referrer to be, well, an Umlaut page, like the one we are
+    # currently generating would be best. That is, the resolve link. 
+    
+    header["Referer"] = "http://#{orig_env['HTTP_HOST']}#{orig_env['REQUEST_URI']}"
+
+    # Proxy X-Forwarded headers. 
+
+    # The original Client's ip, most important and honest. Look for
+    # and add on to any existing x-forwarded-for, if neccesary, as per
+    # x-forwarded-for convention. 
+    header['X-Forwarded-For'] =  (orig_env['HTTP_X_FORWARDED_FOR']) ?
+       (orig_env['HTTP_X_FORWARDED_FOR'].to_s + ', ' + request.remote_ip.to_s) :
+       request.remote_ip.to_s
+       
+    #Theoretically the original host requested by the client in the Host HTTP request header. We're disembling a bit.  
+    header['X-Forwarded-Host'] = host if host
+    # The proxy server: That is, Umlaut, us. 
+    header['X-Forwarded-Server'] = orig_env['SERVER_NAME'] || ''
+
+    return header
+  end
+  
+end

data/app/controllers/search_controller.rb

@@ -0,0 +1,328 @@
+# The search controller handles searches fo manually entered citations,
+# or possibly ambiguous citations generally. It also provides an A-Z list.
+#
+# As a source of this data, it generally talks to the SFX database directly. 
+# The particular method it uses to get this data is defined in a SearchMethod
+# module (app/controllers/search_methods), that gets applied to the controller.
+# Currently Sfx3 direct database or Sfx4 direct database are supported. In
+# either case with database connection info in your database.yml file under
+# sfx_db. 
+# 
+# Future plans include a local database of titles, perhaps loaded from an 
+# external KB. Not done yet.
+#
+# = SearchMethod module implementation
+# A search method is just a ruby module, that will be applied to a controller,
+# that defines two methods:
+#   [#find_by_title]
+#     Takes no arguments, instead use methods in the controller like
+#     #sfx_az_profile, #title_query_param, #search_type_param, #batch_size and
+#     #page to return state.  Returns a two-element array pair, first element
+#     is a list of OpenURL::ContextObject for current batch, send element
+#     is int total hit count. 
+#   [#find_by_group]
+#     Used for clicks on "A", "B" ... "0-9", "Other" links.  Find the group
+#     link clicked on in params[:id].  Use #batch_size and #page for paging.
+#     As in #find_by_title, return two element array, first elememt is array
+#     of OpenURL::ContextObject, second element is total hit count. 
+class SearchController < UmlautController
+  @@search_batch_size = 20
+  @@az_batch_size = 20
+  @@autocomplete_limit = 15
+  
+  layout umlaut_config.search_layout, :except => [ :opensearch, :opensearch_description ]
+
+  before_filter :normalize_params
+  
+  def initialize(*params)
+    super(*params)
+    
+    self.extend( search_method_module  )
+  end
+  
+  def index
+    @page_title = "Journals"  
+    journals()          
+  end  
+  
+  def journals
+    @submit_hash = params["umlaut.display_coins"] ? {:controller=>'resolve', :action=>'display_coins'} : {:controller=>'search', :action=>'journal_search'}
+
+    # Render configed view, if configed, or default     
+    render umlaut_config.lookup!("search_view", "journals")
+  end
+
+  # Not sure if this action actually works or does anything at present. 
+  def books
+     @submit_action = params["umlaut.display_coins"] ? "display_coins" : "index"
+  end
+
+  
+  # @display_results is left as an array of ContextObject objects.
+  # Or, redirect to resolve action for single hit.
+  # O hit also redirects to resolve action, as per SFX behavior--this
+  # gives a catalog lookup and an ILL form for 0-hit. 
+  # param umlaut.title_search_type (aka sfx.title_search) 
+  # can be 'begins', 'exact', or 'contains'. Other
+  # form params should be OpenURL, generally
+  def journal_search
+    @batch_size = batch_size
+    @start_result_num = (page * batch_size) - (batch_size - 1)
+    
+    @search_context_object = context_object_from_params
+    
+    if (! params["rft.object_id"].blank? ||
+        ! params["rft.issn"].blank? ||
+        ! params["rft_id"].blank? )
+      # If we have an exact-type 'search', just switch to 'resolve' action
+      redirect_to url_for_with_co( {:controller => 'resolve'}, context_object_from_params ) 
+      
+      # don't do anything else.
+      return
+    elsif (params['rft.jtitle'].blank?)
+      #Bad, error condition. If we don't have any of that other stuff above,
+      # we need a title!  Send them back to entry page with an error message.
+      flash[:error] = "You must enter a journal title or other identifying information."
+      redirect_to :controller=>:search, :action=>:index
+      return
+    end
+    
+    
+    
+    # Call our particular search method, #find_by_title added by search
+    # method module. 
+    (@display_results, @hits) = self.find_by_title
+    #find_by_title_via_sfx_db
+    
+    # Calculate end-result number for display
+    @end_result_num = @start_result_num + batch_size - 1
+    if @end_result_num > @hits
+      @end_result_num = @hits
+    end
+        
+    if (@page == 1) && (@display_results.length == 1)
+      # If we narrowed down to one result redirect
+      # to resolve action.
+      redirect_to( url_for_with_co({:controller => 'resolve'}, @display_results[0]) )      
+    elsif (@display_results.length == 0)
+      # 0 hits, do it too.
+      redirect_to(  url_for_with_co({:controller => 'resolve'}, @search_context_object) )
+    end
+
+  end
+
+
+  # Used for browse-by-letter 
+  def journal_list
+    @batch_size = batch_size
+    @page = page         
+    @start_result_num = (@page * @batch_size) - (@batch_size - 1)
+
+
+    (@display_results, @hits) = find_by_group                         
+
+
+    # Calculate end-result number for display
+    @end_result_num = @start_result_num + @batch_size - 1
+    if @end_result_num > @hits
+      @end_result_num = @hits
+    end
+    
+    # Use our ordinary search displayer to display
+    # It'll notice the action and do just a bit of special stuff.
+    render(:template => "search/journal_search")
+  end  
+
+  
+  
+  
+
+  # Should return an array of hashes, with each has having :title and :object_id
+  # keys. Can come from local journal index or SFX or somewhere else.
+  # :object_id is the SFX rft.object_id, and can be blank. (I think it's SFX
+  # rft.object_id for local journal index too)
+  def auto_complete_for_journal_title
+   # Don't search on blank query.
+   query = params['rft.jtitle']
+   search_type = params["umlaut.title_search_type"] || "contains"
+   unless ( query.blank? )    
+      (context_objects, total_count) = find_by_title
+      @titles = context_objects.collect do |co| 
+        metadata = co.referent.metadata
+        {:object_id => metadata["object_id"], :title => (metadata["jtitle"] || metadata["btitle"] || metadata["title"])}
+      end
+   end
+   render :text => @titles.to_json, :content_type => "application/json"   
+  end  
+  
+  def opensearch_description
+    @headers['Content-Type'] = 'application/opensearchdescription+xml' 
+  end
+
+  protected
+
+  def normalize_params
+    # citation search params  
+  
+    # sfx.title_search and umlaut.title_search_type are synonyms
+    params["sfx.title_search"] = params["umlaut.title_search_type"] if params["sfx.title_search"].blank?
+    params["umlaut.title_search_type"] = params["sfx.title_search"] if params["umlaut.title_search_type"].blank?
+    
+    # Likewise, params[:journal][:title] is legacy params['rft.jtitle']
+    unless (params[:journal].blank? || params[:journal][:title].blank? ||
+            ! params['rft.jtitle'].blank? )
+      params['rft.jtitle'] = params[:journal][:title]
+    end
+
+    if ( (params[:journal].blank? || params[:journal][:title].blank?) &&
+          params['rft.jtitle'] )
+      params[:journal] ||= {}
+      params[:journal][:title] = params['rft.jtitle']
+    end
+
+    
+    # Grab identifiers out of the way we've encoded em
+    # Accept legacy SFX-style encodings too
+    if ( ! params['rft_id_value'].blank? ||
+        ! params['pmid_value'].blank? || 
+        ! params['doi_value'].blank?  )
+
+      if (! params['rft_id_value'].blank?)
+        id_type = params['rft_id_type'] || 'doi'
+        id_value = params['rft_id_value']
+      elsif (! params['pmid_value'].blank?)
+        id_type = params['pmid_id'] || 'pmid'
+        id_value = params['pmid_value']
+      else # sfx-style doi
+        id_type = params['doi_id'] || 'doi'
+        id_value = params['doi_value']
+      end
+              
+      params['rft_id'] = "info:#{id_type}/#{id_value}"
+    end
+
+    # SFX v2 A-Z list url format---convert to Umlaut
+    if params[:letter_group]
+      params[:id] = case params[:letter_group].to_i        
+        when 1 then '0-9'
+        # 2-27 mean A-Z, convert via ASCII value arithmetic.  
+        when 2..27 then ((params[:letter_group].to_i) +63 ).chr
+        when 28 then 'Others'
+      end
+      params.delete(:letter_group) if params[:id]
+    end
+
+    # SFX v3 A-Z list url format--convert to Umlaut
+    if params[:param_letter_group_value]
+      params[:id] = case params[:param_letter_group_value]
+        when /^0/ then '0-9'
+        when 'Others' then 'Other'
+        else params[:param_letter_group_value]
+      end          
+    end
+
+    # Normalize request for 'Others'
+    if params[:id] =~ /^other/i 
+       params[:id] = 'Others'
+    end
+    
+    # for reasons I can't tell, our JS on IE ends up putting some
+    # newlines in the object_id, which messes us all up.
+    params['rft.object_id'].strip! if params['rft.object_id']
+
+    ## If needed combine date elements to an OpenURL date
+    unless (params["__year"].blank? && 
+            params["__month"].blank? && 
+            params["__day"].blank?)
+      isoDate = ""
+      unless ["", "****", "Year"].include?(params["__year"])
+        isoDate += params["__year"]
+        unless ["", "***", "Month"].include?(params["__month"])
+          isoDate += "-" + params["__month"]
+          unless ["", "**", "Day"].include?(params["__day"])
+            isoDate += "-" + params["__day"]
+          end
+        end
+      end
+      unless isoDate.blank?
+        params["date"] = isoDate
+      end
+    end
+
+  end
+
+  def context_object_from_params
+    @context_object_from_params ||= 
+      begin
+      params_c = params.clone  
+  
+      # Take out the weird ones that aren't really part of the OpenURL
+      ignored_keys = [:journal, "utf8", "__year", "__month", "__day", "action", "controller", "Generate_OpenURL2", "rft_id_type", "rft_id_value"]
+      ignored_keys.each { |k| params_c.delete(k) }
+      
+      # Normalize ISSN to have dash
+      if ( ! params['rft.issn'].blank? && params['rft.issn'][4,1] != '-' && params['rft.issn'].length >= 4)
+        params['rft.issn'].insert(4,'-')
+      end
+    
+      ctx = OpenURL::ContextObject.new
+      # Make sure it uses a journal type referent please, that's what we've
+      # got here.
+      ctx.referent = OpenURL::ContextObjectEntity.new_from_format( 'info:ofi/fmt:xml:xsd:journal' )
+      ctx.import_hash( params_c )
+  
+      # Not sure where ":rft_id_value" as opposed to 'rft_id' comes from, but
+      # it was in old code. We do it after CO creation to handle multiple
+      # identifiers
+      if (! params_c[:rft_id_value].blank?)
+        ctx.referent.add_identifier( params_c[:rft_id_value] )
+      end              
+      ctx
+    end
+  end
+
+  def search_method_module
+    umlaut_config.lookup!("az_search_method", SearchMethods::Sfx4)
+  end
+  
+  # sfx a-z profile as defined in config, used for direct db connections
+  # to sfx. 
+  def sfx_az_profile
+    umlaut_config.lookup!("sfx_az_profile", "default")
+  end
+  helper_method :sfx_az_profile
+  
+  def title_query_param    
+    params['rft.jtitle']
+  end
+  helper_method :title_query_param
+  
+  def search_type_param
+    params['umlaut.title_search_type'] || 'contains'
+  end
+  helper_method :search_type_param
+  
+  def batch_size
+    case params[:action]
+      when "journal_list"
+        @@az_batch_size      
+      when "auto_complete_for_journal_title"
+        @@autocomplete_limit
+      else
+        @@search_batch_size
+    end
+  end
+  helper_method :batch_size
+  
+  def page
+    @page ||= params['page'].blank? ? 1 : params['page'].to_i     
+  end
+  helper_method :page    
+
+
+ 
+
+
+
+ 
+end

data/app/controllers/search_methods/sfx3.rb

@@ -0,0 +1,148 @@
+module SearchMethods
+  module Sfx3      
+    
+    protected
+    
+    #returns pair of 1) array of context object results for current page, 2) hit count
+    def find_by_title
+      (object_ids, hit_count) = object_ids_az_v3(title_query_param, search_type_param, batch_size, page )
+      
+      # Now fetch objects with publication information
+      # Sometimes SFX db lacks referential integrity, so we don't count
+      # on these object_ids actually being there, doing a find all instead
+      # of just a 'find'. 
+      sfx_objects = SfxDb::Object.find( :all, :conditions => {:OBJECT_ID => object_ids},
+      :include => [:publishers, :main_titles, :primary_issns, :primary_isbns])
+      
+      # We got the right set of @batch_size objects, but they're not sorted
+      # by title. 
+      # Too hard to include the sort in the SQL, let's re-sort in memory
+      sfx_objects.sort! do |a,b| 
+        if (a.main_titles.first && b.main_titles.first)
+          a.main_titles.first.TITLE_DISPLAY <=> b.main_titles.first.TITLE_DISPLAY
+        else
+          0
+        end
+      end
+      
+      # Now we need to convert to ContextObjects.
+      context_objects = sfx_objects.collect do |sfx_obj|
+        ctx = OpenURL::ContextObject.new
+        # Start out with everything in the search, to preserve date/vol info
+        ctx.import_context_object( context_object_from_params )
+  
+        # Put SFX object id in rft.object_id, that's what SFX does. 
+        ctx.referent.set_metadata('object_id', sfx_obj.id.to_s)
+  
+        publisher_obj = sfx_obj.publishers.first
+        if ( publisher_obj )
+          ctx.referent.set_metadata('pub', publisher_obj.PUBLISHER_DISPLAY)
+          ctx.referent.set_metadata('place', publisher_obj.PLACE_OF_PUBLICATION_DISPLAY)
+        end
+        
+        title_obj = sfx_obj.main_titles.first
+        title = title_obj ? title_obj.TITLE_DISPLAY : "Unknown Title"
+        ctx.referent.set_metadata('jtitle', title)
+  
+        issn_obj = sfx_obj.primary_issns.first
+        ctx.referent.set_metadata('issn', issn_obj.ISSN_ID) if issn_obj
+  
+        isbn_obj = sfx_obj.primary_isbns.first     
+        ctx.referent.set_metadata('isbn', isbn_obj.ISBN_ID) if isbn_obj
+        
+        ctx
+      end
+      return [context_objects, hit_count]
+    end
+    
+
+  
+  # Object Ids from SFX A-Z list 'version 3'. The A-Z v3 title list
+  # is cranky, so we have a v2 version too. 
+  # input title query, search_type
+  # Returns an array [ batch_obj_id_array, count ]. 
+  def object_ids_az_v3(title_q, search_type, batch_size, page)
+    # MySQL 'like' is case-insensitive, fortunately, don't need to worry
+    # about that. But to deal with non-filing chars, need to search against
+    # TITLE_DISPLAY and TITLE_SORT for begins with. We're going to join
+    # to AZ_TITLE_SEARCH_VER3 for alternate titles too. 
+    conditions = case search_type
+      when 'contains'
+        ['ts.AZ_PROFILE = ? AND TITLE_DISPLAY like ? OR ts.TITLE_SEARCH like ?',
+        sfx_az_profile,
+        "%" + title_q.upcase + "%", "%" + title_q.upcase + "%"]
+      when 'begins'
+       ['ts.AZ_PROFILE = ? AND TITLE_DISPLAY like ? OR TITLE_SORT like ? OR ts.TITLE_SEARCH like ?',
+       sfx_az_profile,
+       title_q + '%', title_q + '%', title_q + "%"]
+      else # exact
+        ['ts.AZ_PROFILE = ? AND TITLE_DISPLAY = ? OR TITLE_SORT =  ? OR ts.TITLE_SEARCH = ?', 
+        sfx_az_profile,
+        title_q, title_q, title_q]
+    end
+    
+    # First get object_ids we're interested in, then
+    # we'll bulk fetch with all their data. 
+    # Tricky-ass query for efficiency and power, sorry.
+    joins = "left outer join AZ_TITLE_SEARCH_VER3 as ts on  `AZ_TITLE_VER3`.AZ_TITLE_VER3_ID = ts.AZ_TITLE_VER3_ID"
+
+    
+    # Actually, _first_ we'll do a total count.
+    total_hits = SfxDb::AzTitle.count(:OBJECT_ID, 
+    :distinct=>true,
+    :conditions=>conditions,
+    :joins=>joins,
+    :order=>'TITLE_SORT ASC')
+
+    # Now fetch object_ids for just the display batch
+    object_ids = SfxDb::AzTitle.find(:all, :select=>"distinct (OBJECT_ID)",
+    :conditions => conditions, 
+    :joins => joins,
+    :limit => batch_size,
+    :offset => batch_size * (page - 1),
+    :order=>'TITLE_SORT ASC').collect { |title_obj| title_obj.OBJECT_ID}
+
+
+    return [ object_ids, total_hits]
+  end
+
+  # params[:id] will have a capital letter, or "0-9" or "Other"
+  def find_by_group
+    
+      joins = " inner join AZ_LETTER_GROUP_VER3 as lg on AZ_TITLE_VER3.AZ_TITLE_VER3_ID = lg.AZ_TITLE_VER3_ID"
+      # Need a special condition for 0-9
+      if ( params[:id] == '0-9')
+        conditions = ["AZ_PROFILE=? AND lg.AZ_LETTER_GROUP_VER3_NAME IN ('0','1','2','3','4','5','6','7','8','9')",
+        sfx_az_profile]
+      else
+        conditions = ["AZ_PROFILE=? AND lg.AZ_LETTER_GROUP_VER3_NAME = ?", 
+        sfx_az_profile,
+        params[:id].upcase]
+      end
+
+      hits = SfxDb::AzTitle.count(:joins => joins,
+            :conditions => conditions)
+
+
+      # Sorry this find is so crazy, trying to manage to do it
+      # efficiently and get what we need in large SFX db. 
+      # For crazy nested include below, see:
+      # http://snippets.dzone.com/posts/show/2089
+      az_titles = SfxDb::AzTitle.find(:all, 
+        :joins => joins,
+        :conditions => conditions,
+        :limit => batch_size,
+        :offset=> batch_size * (page - 1),
+        :order=>'TITLE_SORT',
+        :include=>[{:object => [:publishers, :titles] }])
+        
+      # Convert to context objects
+      display_results = az_titles.collect do | azt | 
+        co = azt.to_context_object
+        co.referrer.add_identifier('info:sid/umlaut.code4lib.org:azlist')
+        co
+      end
+      return [display_results, hits]
+    end
+  end
+end