umlaut 4.0.3 → 4.1.0.pre.2

data/app/helpers/umlaut/section_highlights.rb

@@ -0,0 +1,85 @@
+module Umlaut
+  # NOT Rails helper methods, but a helper class with logic to determine
+  # whether a given umlaut display section should be given the
+  # umlaut-section-highlighted class, used to mark recommended access
+  # methods. (For instance, fulltext if it's available, or maybe
+  # document_delivery if it's not, but it gets more complicated.)
+  class SectionHighlights
+    attr_reader :umlaut_request, :umlaut_config
+
+    # * First arg is the umlaut Request
+    # * Second optional is an UmlautConfiguration object, used for 
+    #   `section_highlights_filter` lambda -- will default to
+    #   UmlautController.umlaut_config
+    def initialize(umlaut_request, umlaut_config = UmlautController.umlaut_config)
+      @umlaut_config  = umlaut_config
+      @umlaut_request = umlaut_request
+    end
+
+    def should_highlight_section?(section_id)
+      highlighted_sections.include? section_id.to_s
+    end
+
+    # array of section div_id's that should be highlighted for
+    # the current request in it's current state. Calculated
+    # with calc_highlighted_sections!, then cached. 
+    def highlighted_sections
+      @highlighted_sections ||= calc_highlighted_sections!
+    end
+
+    # Returns an array of zero or more sections to display with 
+    # .umlaut-section-highlighted -- usually the recommended section,
+    # fulltext if we have it, etc. 
+    #
+    # A bit hard to get exactly right for both technical and contextual
+    # policy issues, this is a basic starting point. 
+    def calc_highlighted_sections!
+      sections = []
+
+      if umlaut_request.get_service_type("fulltext").present?
+        sections << "fulltext"
+      end
+
+      # Highlight holdings if it's present AND:
+      #   no fulltext is present OR it's a book (non-serial) type
+      # We think people want print for books more often. 
+      if umlaut_request.get_service_type("holding").present? &&
+         ( umlaut_request.get_service_type("fulltext").blank? ||  (! MetadataHelper.title_is_serial?(umlaut_request.referent)) )
+        sections << "holding"
+      end
+
+
+      # Return document_delivery as highlighted only if 
+      # fulltext and holdings are done being fetched. AND. 
+      # If there's no fulltext or holdings, OR there's holdings, but
+      # it's a journal type thing, where we probably don't know if the
+      # particular volume/issue wanted is present. Ugh. 
+      if ( umlaut_request.get_service_type("document_delivery").present? &&
+           umlaut_request.get_service_type("fulltext").empty? && 
+            (! umlaut_request.service_types_in_progress?(["fulltext", "holding"])) && (
+              umlaut_request.get_service_type("holding").empty? || 
+              umlaut_request.referent.format == "journal"
+            )
+          )
+        sections << "document_delivery"
+      end
+
+      sections = apply_filters!(sections)
+
+      return sections
+    end
+
+
+    def apply_filters!(sections)
+      sections = sections.dup
+
+      (umlaut_config.section_highlights_filter || []).each do |filter|
+        # filters are expected to mutate 'sections' if they want
+        filter.call(umlaut_request, sections, self)
+      end
+
+      return sections
+    end
+
+  end
+end

data/app/mixin_logic/metadata_helper.rb

@@ -74,7 +74,7 @@ module MetadataHelper
     title.gsub!(/\&/, ' and ') if options[:normalize_ampersand]
       
     # remove non-alphanumeric, excluding apostrophe
-    title.gsub!(/[^\w\s\']/, ' ') if options[:remove_punctuation]
+    title.gsub!(/[^[[:alnum:]][[:space:]]\']/, ' ') if options[:remove_punctuation]
 
     # apostrophe not to space, just eat it.
     title.gsub!(/[\']/, '') if options[:remove_punctuation] && ! options[:keep_apostrophes]
@@ -341,10 +341,16 @@ module MetadataHelper
   # Look at weird bad OpenURLs, use heuristics to see if the 'title' probably
   # represents a journal rather than a book. A guess at best, based on the bad
   # data we've seen, sigh. 
-  def title_is_serial?(rft)    
-    rft.format != "book" &&
-    ( rft.metadata['jtitle'].present? || %w{journal article}.include?(rft.metadata["genre"]) ) &&
-    rft.metadata['btitle'].blank?
+  def title_is_serial?(rft)   
+    ( rft.format != "book" && rft.format != "dissertation") &&
+    (  rft.metadata["btitle"].blank?  ) &&
+    ( %w{journal article}.include?(rft.metadata["genre"]) ||
+      rft.metadata['jtitle'].present? ||
+      (rft.metadata["genre"].blank? && rft.metadata["issn"].present?)
+    )  
   end
+  # Mark it a module function so it can be called as a utility as
+  # MetadataHelper.title_is_serial?(referent)
+  module_function :title_is_serial?
   
 end

data/app/models/collection.rb

@@ -12,6 +12,8 @@ require 'confstruct'
 # The Collection holds and executes the logic for running those services,
 # foreground and background, making sure no service is run twice if it's
 # already in progress, timing out expired services, etc.
+#
+# This code is a mess, sorry. 
 class Collection
   attr_accessor :umlaut_request
   attr_accessor :logger
@@ -66,26 +68,22 @@ class Collection
   #
   # Returns the Thread object used for dispatching background services
   def dispatch_services!
-    queued_service_ids = prepare_for_dispatch!
-
-    dispatch_foreground!(queued_service_ids)
+    freshen_dispatches!
+    mark_queued_if_empty!
+    
+    dispatch_foreground!
 
     # return main thread for background services.
-    return dispatch_background!(queued_service_ids)
+    return dispatch_background!
   end
 
-  # Call prepare_for_dispatch! first, the return value from that call
-  # is suitable as argument for this call: queued_service_ids, list of
-  # service id's already identified as suitable for running, and
-  # marked queued in the DispatchedService table.
-  #
   # Will run such services in foreground priority waves. And then reload
   # the UmlautRequest object in the current thread, to pick up any
   # changes made in service threads.
-  def dispatch_foreground!(queued_service_ids)
+  def dispatch_foreground!
     # Foreground services
     (0..9).each do | priority |
-      services_to_run = self.instantiate_services!(:level => priority, :ids => queued_service_ids)
+      services_to_run = self.instantiate_services!(:level => priority, :ids => runnable_services_for_priority(priority))
       next if services_to_run.empty?
       ServiceWave.new(services_to_run , priority).handle(umlaut_request, umlaut_request.session_id)
     end
@@ -96,15 +94,13 @@ class Collection
     umlaut_request.reload
   end
 
-  # Call prepare_for_dispatch! first, the return value from that call
-  # is suitable as argument for this call: queued_service_ids, list of
-  # service id's already identified as suitable for running, and
-  # marked queued in the DispatchedService table.
-  #
-  # Will run such services in background priority waves.
+  # Will run such services in background priority waves. If some
+  # services are already running, will not run services in subsequent
+  # waves until they are done -- guard against multiple HTTP
+  # requests while services in progress. 
   #
   # Returns the Thread object used for dispatching background services. 
-  def dispatch_background!(queued_service_ids)
+  def dispatch_background!
     # Now we do some crazy magic, start a Thread to run our background
     # services. We are NOT going to wait for this thread to join,
     # we're going to let it keep doing it's thing in the background after
@@ -124,11 +120,22 @@ class Collection
         # other stuff done before this thread.
         Thread.pass
 
+        force_refresh = false
 
         ('a'..'z').each do | priority |
-            services_to_run = self.instantiate_services!(:level => priority, :ids => queued_service_ids)
-          next if services_to_run.empty?
+          # force refresh only if we just ran some services, otherwise not enough
+          # time has gone by to be worthwhile. 
+          runnable_ids = runnable_services_for_priority(priority, :refresh => force_refresh)
+
+          services_to_run = self.instantiate_services!(:level => priority, :ids => runnable_ids)
+          
+          if services_to_run.empty?
+            force_refresh = false
+            next
+          end
+
           ServiceWave.new(services_to_run , priority).handle(umlaut_request, umlaut_request.session_id)
+          force_refresh = true
         end
       rescue Exception => e
         # We are divorced from any HTTP request at this point, and may not
@@ -154,90 +161,116 @@ class Collection
     end
   end
 
-
-  # Goes through all services and marks them with a DispatchedService
-  # record in 'queued' state.
-  #
-  # Will time out any too-old services in a running state.
-  #
-  # Will remove DispatchedService status for
-  # any services marked failed that are old enough to re-run, or services
-  # that are too old to re-use.   Such services are then queuable.
-  #
-  # Returns array of Service identifiers for services that are now
-  # queued and execable.
-  def prepare_for_dispatch!
-    # Go through currently dispatched services, looking for timed out
-    # services -- services still in progress that have taken too long,
-    # as well as service responses that are too old to be used.
-
-    queued_service_ids = []
-    DispatchedService.transaction do
-      umlaut_request.dispatched_services.each do | ds |
-          # go through dispatched_services and set stil in progress but too long to failed temporary
-          if ( (ds.status == DispatchedService::InProgress ||
-                ds.status == DispatchedService::Queued ) &&
-                (Time.now - ds.updated_at) > self.background_service_timeout)
-
-                ds.store_exception( Exception.new("background service timed out (took longer than #{self.background_service_timeout} to run); thread assumed dead.")) unless ds.exception_info
-                # Fail it temporary, it'll be run again.
-                ds.status = DispatchedService::FailedTemporary
-                ds.save!
-                logger.warn("Background service timed out, thread assumed dead. #{umlaut_request.id} / #{ds.service_id}")
-           end
-
-
-
-          # go through dispatched_services and delete:
-          # 1) old completed dispatches, too old to use.
-          # 2) failedtemporary dispatches that are older than our resurrection time
-          # -> And all responses associated with those dispatches.
-          # After being deleted, they'll end up re-queued.
-          if ( (ds.completed? && completed_dispatch_expired?(ds) ) ||
-               (  ds.status == DispatchedService::FailedTemporary &&
-                 (Time.now - ds.updated_at) > self.requeue_failedtemporary_services_in
-                )
-              )
-
-            # Need to expire. Delete all the service responses, and
-            # the DispatchedService record, and service will be automatically
-            # run again.
-            serv_id = ds.service_id
-
-            umlaut_request.service_responses.each do |response|
-              if response.service_id == serv_id
-                umlaut_request.service_responses.delete(response)
-                response.destroy
-              end
-            end
-
-            umlaut_request.dispatched_services.destroy(ds)            
+  # Goes through existing DispatchedService objects, and freshens them up:
+  # * If a service is marked in progress longer than timeout, mark
+  #   it failed temporary.
+  # * If an existing failed temporary is older than our resurrection time,
+  #   delete the dispatch (and all it's responses), so it can be re-queued. 
+  def freshen_dispatches!
+    umlaut_request.dispatched_services.each do | ds |
+      # go through dispatched_services and set still in progress but too long to failed temporary
+      if ( (ds.status == DispatchedService::InProgress ||
+            ds.status == DispatchedService::Queued ) &&
+            (Time.now - ds.updated_at) > self.background_service_timeout)
+
+            ds.store_exception( Exception.new("background service timed out (took longer than #{self.background_service_timeout} to run); thread assumed dead.")) unless ds.exception_info
+            # Fail it temporary, it'll be run again.
+            ds.status = DispatchedService::FailedTemporary
+            ds.save!
+            logger.warn("Background service timed out, thread assumed dead. #{umlaut_request.id} / #{ds.service_id}")
+       end
+
+
+
+      # go through dispatched_services and delete:
+      # 1) old completed dispatches, too old to use.
+      # 2) failedtemporary dispatches that are older than our resurrection time
+      # -> And all responses associated with those dispatches.
+      # After being deleted, they'll end up re-queued.
+      if ( (ds.completed? && completed_dispatch_expired?(ds) ) ||
+           (  ds.status == DispatchedService::FailedTemporary &&
+             (Time.now - ds.updated_at) > self.requeue_failedtemporary_services_in
+            )
+          )
+
+        # Need to expire. Delete all the service responses, and
+        # the DispatchedService record, and service will be automatically
+        # run again.
+        serv_id = ds.service_id
+
+        umlaut_request.service_responses.each do |response|
+          if response.service_id == serv_id
+            umlaut_request.service_responses.delete(response)
+            response.destroy
           end
+        end
+
+        umlaut_request.dispatched_services.destroy(ds)            
       end
+    end
+  end
+
+  # For all configured services, if they have NO DispatchedService
+  # object, then create one with status Queued
+  def mark_queued_if_empty!
+    our_service_ids = self.get_service_definitions.collect {|d| d["service_id"]}
+
+    existing_dispatches = umlaut_request.dispatched_services.collect {|d| d.service_id}
+
+    not_yet_existing = our_service_ids - existing_dispatches
 
-      # Queue any services without a dispatch marker at all, keeping
-      # track of queued services, already existing or newly created.
-
-      # Just in case, we're going to refetch dispatched_services from the db,
-      # in case some other http request or background service updated things
-      # recently.
-      umlaut_request.dispatched_services.reset
-
-      self.get_service_definitions.each do |service|
-        service_id = service['service_id']
-        # use in-memory #to_a search, don't go to db each time!
-        if found = umlaut_request.dispatched_services.to_a.find {|s| s.service_id == service_id}
-          queued_service_ids.push(service_id) if found.status == DispatchedService::Queued
-        else
-          umlaut_request.new_dispatch_object!(service_id, DispatchedService::Queued).save!
-          queued_service_ids.push(service_id)
+    not_yet_existing.each do |service_id|
+      umlaut_request.new_dispatch_object!(service_id, DispatchedService::Queued).save!
+    end    
+  end
+
+  # All services for priority that are marked Queued, so long as
+  # no previous waves are still marked running. 
+  #
+  # Pass `:refresh => true` as second argument to force trip
+  # to the database to get fresh DispatchedService objects. 
+  #
+  # Returns array of service_id's, or empty array. 
+  def runnable_services_for_priority(priority, options = {})
+    DispatchedService.connection_pool.with_connection do
+      service_definitions = self.get_service_definitions
+
+      # Make a hash where key is service id, and value is priority.to_s
+      service_to_priority = Hash[
+        service_definitions.collect do |d|
+          [ d["service_id"], d["priority"].to_s ]
         end
+      ]
+
+      if options[:refresh]
+        # force a refresh
+        umlaut_request.dispatched_services(true)
       end
-    end
 
-    return queued_service_ids
+      # If there is any service earlier than this wave still marked InProgress, 
+      # we're not ready to run this wave, return empty array. 
+      # Important to avoid race condition on HTTP requests, don't
+      # dispatch later background waves unless earlier are actually complete,
+      # even on an HTTP status check. 
+      previous_waves_running = umlaut_request.dispatched_services.find do |ds|
+        ds.status == DispatchedService::InProgress &&
+        service_to_priority[ ds.service_id ] < priority.to_s
+      end.present?
+      return [] if previous_waves_running
+
+      # otherwise, the services for this priority are runnable if
+      # they are already marked Queued
+      # We use .to_a, we want to use the already in memory array, not
+      # go to the db here. 
+      return umlaut_request.dispatched_services.to_a.find_all do |ds|
+        ds.status == DispatchedService::Queued &&
+        service_to_priority[ ds.service_id ] == priority.to_s
+      end.collect {|ds| ds.service_id}
+    end
   end
 
+
+
   def completed_dispatch_expired?(ds)
     interval = self.response_expire_interval
     crontab = self.response_expire_crontab_format

data/app/models/dispatched_service.rb

@@ -68,4 +68,8 @@ class DispatchedService < ActiveRecord::Base
   def completed?
     return (self.status != InProgress) && (self.status != Queued)
   end
+
+  def failed?
+    return (self.status == FailedTemporary) || (self.status == FailedFatal)
+  end
 end

data/app/models/referent.rb

@@ -390,11 +390,17 @@ class Referent < ActiveRecord::Base
 
   # options => { :overwrite => false } to only enhance if not already there
   def enhance_referent(key, value, metadata=true, private_data=false, options = {})
+
+
     ActiveRecord::Base.connection_pool.with_connection do
       return if value.nil?
   
       matches = self.referent_values.to_a.find_all do |rv| 
-        (rv.key_name == key) && (rv.metadata == metadata) && (rv.private_data == private_data) 
+        # We ignore #metadata and #private_data matches in overwriting
+        # existing value. We used to take them into account, but it triggered
+        # a bug in Jruby, and pretty much isn't neccesary, those fields
+        # are pretty useless and mostly not used and should prob be removed. 
+        (rv.key_name == key) # && (rv.metadata == metadata) && (rv.private_data == private_data) 
       end
       
       matches.each do |rv|

data/app/models/request.rb

@@ -387,6 +387,25 @@ class Request < ActiveRecord::Base
     self.dispatched_services << ds
     return ds
   end
+
+  # Returns an array of 0 or more ServiceDispatch objects matching
+  # specified conditions. Right now only one condition is supported:
+  #
+  #    dispatch_objects_with(:service_type_values => values)
+  #      values can be one or more string names of service types, returns
+  #      DispatchedServices for services whose generated values include
+  #      one or more of what you specified. 
+  def dispatch_objects_with(options = {})
+    value_names = Array(options[:service_type_values])
+
+    raise ArgumentError.new("Need to supply a :service_type_values argument") unless value_names.present?
+
+    list = self.dispatched_services.to_a.find_all do |ds|
+      (value_names & ds.service.service_types_generated.collect(&:name)).present?
+    end
+
+    return list
+  end
   
   protected
 
@@ -465,6 +484,8 @@ class Request < ActiveRecord::Base
     return self.dispatched_services.where(:service_id => service.service_id).first
   end
 
+
+
   # Input is a CGI::parse style of HTTP params (array values)
   # output is a string "fingerprint" canonically representing the input
   # params, which can be stored in the db, so that when another request

data/app/models/service_store.rb

@@ -33,12 +33,19 @@ class ServiceStore
   end
 
 
-  # Returns complete hash loaded from services.yml
+  # Returns complete hash loaded from config/umlaut_services.yml
+  # Passes through ERB first, allowing ERB in umlaut_services.yml
   def config
     # cache hash loaded from YAML, ensure it has the keys we expect.
     unless defined? @services_config_list
       yaml_path = File.expand_path("config/umlaut_services.yml", Rails.root)
-      @services_config_list = (File.exists? yaml_path) ? YAML::load(File.open(yaml_path)) : {}
+
+      @services_config_list = if File.exists? yaml_path 
+        YAML::load(ERB.new(File.open(yaml_path).read).result)
+      else
+        {}
+      end
+
       @services_config_list["default"] ||= {}
       @services_config_list["default"]["services"] ||= {}
     end
@@ -120,13 +127,19 @@ class ServiceStore
   # pass in string unique key OR a service definition hash,
   # and a current UmlautRequest.
   # get back instantiated Service object.
+  #
+  # If string service_id is passed in, but is not defined in application services,
+  # a ServiceStore::NoSuchService exception will be raised. 
   def instantiate_service!(service, request)
     definition = service.kind_of?(Hash) ? service : service_definition_for(service.to_s)
-    raise "Service '#{service}'' does not exist in umlaut-services.yml" if definition.nil?
+    raise NoSuchService.new("Service '#{service}'' does not exist in umlaut-services.yml") if definition.nil?
     className = definition["type"] || definition["service_id"]
     classConst = Kernel.const_get(className)
     service = classConst.new(definition)
     service.request = request
     return service
   end
+
+  class NoSuchService < RuntimeError ; end
+
 end

data/app/presentation/section_renderer.rb

@@ -236,7 +236,6 @@
 #    {:div_id => "search_inside", :partial => "search_inside", :show_partial_only => true}
 class SectionRenderer
   include ActionView::Helpers::TagHelper
-  @@bg_update_sections = @@partial_update_sections = nil
 
   # First argument is the current umlaut Request object.
   # Second argument is a session description hash. See class overview
@@ -266,12 +265,12 @@ class SectionRenderer
   def initialize(a_umlaut_request, section_def = {})
     @umlaut_request = a_umlaut_request
     
-    @section_id = section_def[:id] || section_def[:div_id]
-    raise Exception.new("SectionRenderer needs an :id passed in arguments hash") unless @section_id
+    @div_id = section_def[:div_id] || section_def[:id]
+    raise Exception.new("SectionRenderer needs a :div_id passed in arguments hash") unless @div_id
+    
 
     # Merge in default arguments for this section from config. 
     construct_options(section_def)
-
   end
 
   # Returns all ServiceTypeValue objects contained in this section, as
@@ -319,7 +318,7 @@ class SectionRenderer
   end
 
   def div_id
-    return @section_id
+    return @div_id
   end
 
   def show_heading?
@@ -341,7 +340,7 @@ class SectionRenderer
 
     { :partial => "background_progress",
       :locals =>{ :svc_types => service_type_values,
-                  :div_id => "progress_#{@section_id}",
+                  :div_id => "progress_#{self.div_id}",
                   :current_set_empty => responses_empty?,
                   :item_name => custom_item_name
                 }
@@ -404,7 +403,7 @@ class SectionRenderer
       when :complete_with_responses
         (! responses.empty?) && ! (services_in_progress?)
       when Proc
-        # It's a lambda, which takes @umlaut_request as an arg
+        # It's a lambda, which takes this SectionRenderer as an arg
         @options[:visibility].call(self)
       else true        
     end
@@ -504,20 +503,20 @@ class SectionRenderer
     
 
     # service type value default to same name as section_id
-    @options[:service_type_values] ||= [@section_id]    
+    @options[:service_type_values] ||= [self.div_id]    
 
     # Partials to display. Default to _standard_response_item item partial.
     if ( @options[:partial] == true)
-      @options[:partial] = @section_id
-    end
-    if (@options[:partial].blank?)
-      @options[:item_partial] = 
-        case @options[:item_partial]
-          when true then @section_id + "_item"
-          when String then options[:item_partial]
-          else "standard_response_item"
-        end
+      @options[:partial] = self.div_id
     end
+    
+    @options[:item_partial] = 
+      case @options[:item_partial]
+        when true then self.div_id + "_item"
+        when String then options[:item_partial]
+        else "standard_response_item"
+      end
+  
 
     # sanity check
     if ( @options[:show_partial_only] && ! @options[:partial])

data/app/service_adaptors/blacklight.rb

@@ -90,6 +90,7 @@ class Blacklight < Service
       holdings_url = blacklight_precise_search_url( request, "dlf_expanded" )
       holdings_added += add_holdings( holdings_url ) if holdings_url
     end
+
     #keyword search.    
     if (@keyword_search &&
         url = blacklight_keyword_search_url(request))
@@ -210,8 +211,8 @@ class Blacklight < Service
     # phrase search for title, just raw dismax for author
     # Embed quotes inside the quoted value, need to backslash-quote for CQL,
     # and backslash the backslashes for ruby literal. 
-    clauses.push("#{@bl_fields["title"]} = \"\\\"#{remove_quotes_for_phrase_value title}\\\"\"")    
-    clauses.push("#{@bl_fields["author"]} = \"#{remove_quotes_for_phrase_value author}\"") if author
+    clauses.push("#{@bl_fields["title"]} = \"\\\"#{escape_for_cql_double_quotes title}\\\"\"")    
+    clauses.push("#{@bl_fields["author"]} = \"#{escape_for_cql_double_quotes author}\"") if author
     
     url = base_url + "?search_field=#{@cql_search_field}&content_format=#{options[:content_format]}&q=#{CGI.escape(clauses.join(" AND "))}"
 
@@ -228,8 +229,14 @@ class Blacklight < Service
   # error if we do nothing. Can we escape it somehow? CQL is really
   # unclear, we're ALREADY backslash escaping the phrase quotes themselves!
   # We just replace them with space, should work for our actual indexing. 
-  def remove_quotes_for_phrase_value(str)
-    str.gsub('"', " ")
+  #
+  # Single quotes (apostrophes) need to be escaped with an apostrophe itself,
+    # `''`, apparently. http://mail-archives.apache.org/mod_mbox/cassandra-user/201108.mbox/%3C20110803152250.294300@gmx.net%3E
+  def escape_for_cql_double_quotes(str)
+    str = str.gsub('"', " ")
+    str = str.gsub("'", "''")
+
+    return str
   end