umlaut 3.3.1 → 4.0.0.beta1

data/app/models/referent_value.rb

@@ -1,15 +1,16 @@
 class ReferentValue < ActiveRecord::Base
-  attr_accessible :key_name, :value, :normalized_value, :metadata, :private_data
-  belongs_to :referent, :include => :referent_values
+  if Rails::VERSION::MAJOR >= 4
+    belongs_to :referent, lambda { includes :referent_values }
+  else
+    belongs_to :referent, :include => :referent_values
+  end
 
   # Class method to normalize a string for normalized_value attribute. 
   # Right now normalization is just downcasing. Only
   # metadata values should be normalized (ie, not 'identifier' or 'format').
   # identifier and format shoudl be stored in normalized_value unchanged.
   def self.normalize(input)
-      # 'mb_chars' is neccesary for unicode.
-      # normalized_value column only holds 254 bytes.. 
-      return input.mb_chars.downcase.to_s[0..254]
+      return input.scrub.downcase.to_s[0..254]
   end
   
 end

data/app/models/request.rb

@@ -14,11 +14,17 @@ class Request < ActiveRecord::Base
   # added to the db comes first. Less confusing to have a consistent order.
   # Also lets installation be sure services run first will have their
   # responses show up first
-  has_many :service_responses, :order => 'id ASC' 
+  if Rails::VERSION::MAJOR >= 4
+    has_many :service_responses, lambda { order('id ASC') }
+    belongs_to :referent, lambda { includes(:referent_values) }
+  else
+    # Get rid of this when we stop supporting Rails3
+    has_many :service_responses, :order => 'id ASC'
+    belongs_to :referent, :include => :referent_values
+  end
 
   has_many :clickthroughs
 
-  belongs_to :referent, :include => :referent_values
   # holds a hash representing submitted http params
   serialize :http_env
 
@@ -170,7 +176,7 @@ class Request < ActiveRecord::Base
   # marked as Queued, or Failed---otherwise it should be already working,
   # or done. 
   def can_dispatch?(service)
-    ds= self.dispatched_services.find(:first, :conditions=>{:service_id => service.service_id})
+    ds= self.dispatched_services.where(:service_id => service.service_id).first
     
     return ds.nil? || (ds.status == DispatchedService::Queued) || (ds.status == DispatchedService::FailedTemporary)        
   end
@@ -231,22 +237,20 @@ class Request < ActiveRecord::Base
   #
   # Safe to call in thread, uses connection pool checkout. 
   def add_service_response(response_data)
+
     raise ArgumentError.new("missing required `:service` key") unless response_data[:service].kind_of?(Service)
     raise ArgumentError.new("missing required `:service_type_value` key") unless response_data[:service_type_value]
     
     svc_resp = nil
     ActiveRecord::Base.connection_pool.with_connection do
-      svc_resp = ServiceResponse.new
-
+      svc_resp = self.service_responses.build
       
       svc_resp.service_id = response_data[:service].service_id
       response_data.delete(:service)
   
       type_value =  response_data.delete(:service_type_value)
       type_value = ServiceTypeValue[type_value.to_s] unless type_value.kind_of?(ServiceTypeValue)      
-      svc_resp.service_type_value = type_value
-  
-      svc_resp.request = self
+      svc_resp.service_type_value = type_value  
       
       # response_data now includes actual key/values for the ServiceResponse
       # send em, take_key_values takes care of deciding which go directly
@@ -262,9 +266,9 @@ class Request < ActiveRecord::Base
 
   # Methods to look at status of dispatched services
   def failed_service_dispatches
-    return self.dispatched_services.find(:all, 
-      :conditions => ['status IN (?, ?)', 
-      DispatchedService::FailedTemporary, DispatchedService::FailedFatal])
+    return self.dispatched_services.where(
+      :status => [DispatchedService::FailedTemporary, DispatchedService::FailedFatal]
+    ).to_a
   end
 
   # Returns array of Services in progress or queued. Intentionally
@@ -338,23 +342,32 @@ class Request < ActiveRecord::Base
   # :refresh=>true will force a trip to the db to get latest values.
   # otherwise, association is used.  
   def get_service_type(svc_type, options = {})    
+
+
     svc_type_obj = (svc_type.kind_of?(ServiceTypeValue)) ? svc_type : ServiceTypeValue[svc_type]
 
-    if ( options[:refresh])
+    responses = if ( options[:refresh])
       ActiveRecord::Base.connection_pool.with_connection do
-        return self.service_responses.find(:all,
-                                :conditions =>
-                                  ["service_type_value_name = ?",
-                                  svc_type_obj.name ]   
-                                )
+        self.service_responses.where(["service_type_value_name = ?", svc_type_obj.name ]).to_a
       end
     else
       # find on an assoc will go to db, unless we convert it to a plain
-      # old array first.
-      
-      return self.service_responses.to_a.find_all { |response|
+      # old array first.      
+      self.service_responses.to_a.find_all { |response|
         response.service_type_value == svc_type_obj }      
     end
+
+    # Filter out any services with ID's not currently registered in
+    # ServiceStore    
+    (responses, excluded_responses) = responses.partition do |r|
+      ServiceStore.service_definition_for(r.service_id).present?
+    end    
+    if excluded_responses.present?
+      Rails.logger.warn("ServiceResponses skipped for unknown service_ids: " + 
+        excluded_responses.collect {|s| s.service_id}.uniq.join(","))
+    end
+    
+    return responses
   end
   
   
@@ -393,7 +406,7 @@ class Request < ActiveRecord::Base
       
     rft = nil
     if ( params['umlaut.referent_id'])
-       rft = Referent.find(:first, :conditions => {:id => params['umlaut.referent_id']})
+       rft = Referent.where(:id => params['umlaut.referent_id']).first
     end
 
    
@@ -427,7 +440,7 @@ class Request < ActiveRecord::Base
   end
 
   def find_dispatch_object(service)
-    return self.dispatched_services.find(:first, :conditions=>{:service_id => service.service_id})
+    return self.dispatched_services.where(:service_id => service.service_id).first
   end
 
   # Input is a CGI::parse style of HTTP params (array values)

data/app/models/service_response.rb

@@ -1,34 +1,109 @@
 =begin rdoc
 
-A ServiceResponse is a piece of data generated by a Service. It usually will
-be displayed on the resolve menu.
-
-ServiceResponses have a service type, represented by a string. When displaying, ServiceResponses are typically grouped into lists by service type. ServiceResponses are tied to the Service that created them, with the #service accessor.
-
-ServiceResponses have a few basic attributes stored in columns in the db: 'display_text' is the text to put in the hyperlink. 'notes' is available for longer explanatory text (\n in notes will be converted to <br> by view). 'url' can be used to store the url to link to (but see below on linking mechanism). 
+A ServiceResponse represents a single piece of data or content generated in response to a request. 
+For instance, a full text link, a 'see also' link, a cover image, a library holding, or an
+available 'search inside' service. 
+
+ServiceResponses were generated *for* to a particular request (`#request`), were
+generated *by* a particular service (`#service_id` or `#service`), and 
+belong to a *category or type* such as 'fulltext', 'excerpts', or 'search_inside'
+(#service_type_value)
+
+A ServiceResponse additionally has a hash of keys and values representing
+it's actual payload or content or data. Some of these keys are stored 
+individually in database columns of their own, others are stored in
+a hash serialized to attribute `service_data` -- ServiceResponse offers
+an API so you ordinarily don't need to care which is which; any arbitrary
+key/value pairs can be included in a ServiceResponse. 
+
+      service_response.view_data
+      #=> A hash of key/value pairs representing the payload
+
+You ordinarily create a ServiceResponse by asking a particular
+Umlaut Request to do so:
+
+      umlaut_request.add_service_response(
+        :service=>self, 
+        :service_type_value => :fulltext
+        :display_text => "Some link", 
+        :url => "http://example.com"        
+      )
+
+There are certain conventional keys (like :display_text and :notes) accross
+most response types, and other keys that might be particular to a certain type
+or even a certain service. Services can store data in their own custom keys
+for their own use. See "Conventional Keys" below for documentation on Umlaut
+standard naming. 
+
+== Transformation of view data on display: transform_view_data and response_url
+
+When a ServiceResponse is created, it's initialized with some key/value
+service_data that is stored in the database. 
+
+However, there are also features in place letting a Service filter or transform
+this key/value data at the point of display or use. 
+
+If a Service implements a method `transform_view_data(hash)`, then this method
+will be called at the point of view_data use (possibly multiple times). You can
+modify the hash passsed in, or even replace it entirelyl, just return a new hash.
+
+    def transform_view_data(hash)
+      if hash[:add_some_notes]
+         hash[:display_text] = some_helper_method( params[:something])
+      end
 
-[The legacy columns response_key, value_string, value_alt_string and value_text are deprecated and should not be used, but some legacy Services still use them, so they're still there for now].
+      return hash
+    end
 
-In addition, there's a Hash (automatically serialized by ActiveRecord) that's stored in service_data, for arbitrary additional data that a Service can store--whatever you want, just put it in. However, there are conventions that Views expect, see below.  You can access ALL the arbitrary key/values in a ServiceResponse, built-in in attributes or from the serialized Hash, by the proxy object returned from #data_values.
+This can be especially useful for internationalization (i18n), please see Umlaut
+wiki at https://github.com/team-umlaut/umlaut/wiki/Localization#services-and-their-generated-serviceresponses
 
-You can create a ServiceResponse object with ServiceResponse.create_from_hash, 
-where the hash keys may be direct iVar columns, or serialized in the
-service_data hash, you don't have to care. 
+Similarly, instead of calculating the URL at the point of recording a ServiceResponse,
+you can implement a method `response_url` that will be called when a user clicks
+on a ServiceResponse, and can return a URL to redirect the user too. 
 
-ServiceResponse is connected to a Request via the ServiceType join table. The data architecture allows a ServiceResponse to be tied to multiple requests, perhaps to support some kind of cacheing re-use in the future. But at present, the code doesn't do this, a ServiceResponse will really only be related to one request. However, a ServiceResponse can be related to a single Request more than once--once per each type of service response. ServiceType is really a three way join, representing a ServiceResponse, attached to a particular Request, with a particular ServiceTypeValue.  
+This is useful if the URL is expensive to calculate; or if you want to log or report
+something on click; or if the URL needs user-input to be calculated (eg search_inside deep link)
 
+    def response_url(service_response, submitted_params)
+       return "http://books.google.com/#{service_response.view_data[:book_id]?query=submitted_params[:q]"
+    end
 
 == View Display of ServiceResponse
 
-The resolve menu View expects a Hash (or Hash-like) object with certain conventional keys, to display a particular ServiceResponse. You can provide code in your Service to translate a ServiceResponse to a Hash. But you often don't need to, you can use the proxy object returned by #data_values instead, which provides hash-like access to all arbitrary key/values stored in ServiceResponse. If the Service stores properties in there using conventional keys (see below), no further translation is needed.
+The resolve menu View expects a Hash (or Hash-like) object with certain conventional keys, 
+to display a particular ServiceResponse. You can provide code in your Service to translate a
+ ServiceResponse to a Hash. But you often don't need to, you can use the proxy object returned by 
+ #data_values instead, which provides hash-like access to all arbitrary key/values stored in ServiceResponse. 
+ If the Service stores properties in there using conventional keys (see below), no further translation is needed.
 
 However, if you need to do further translation you can implement methods on the Service, of the form: "to_[service type string](response)", for instance "to_fulltext". Umlaut will give it a ServiceResponse object, method should return a hash (or hash-like obj).  Service can also implement a method response_to_view_data(response), as a 'default' translation. This mechanism of various possible 'translations' is implemented by Service#view_data_from_service_type.
 
 == Url generation
 
-At the point the user clicks on a ServiceResponse, Umlaut will attempt to find a url for the ServiceResponse, by calling response_url(response) on the relevant Service. The default implementation in service.rb just returns service_response['url'], so the easiest way to do this is just to put the url in service_response['url'].  However, your Service can over-ride this method to provide it's own implementation to generate to generate the url on demand in any way it wants.  If it does this, technically service_response['url'] doesn't need to include anything. But if you have a URL, you may still want to put it there, for Umlaut to use in guessing something about the destination, for de-duplication and possibly other future purposes.  
+At the point the user clicks on a ServiceResponse, Umlaut will attempt to find a url for the ServiceResponse, 
+by calling response_url(response) on the relevant Service. The default 
+implementation in service.rb just returns service_response['url'], so the easiest way 
+to do this is just to put the url in service_response['url'].  However, your Service can over-ride 
+this method to provide it's own implementation to generate to generate the url on demand in any way it wants. 
+ If it does this, technically service_response['url'] doesn't need to include anything. But if you have a URL, 
+ you may still want to put it there, for Umlaut to use in guessing something about the destination, 
+ for de-duplication and possibly other future purposes.  
+
+== Note on ServiceType join table. 
+
+ServiceResponse is connected to a Request via the ServiceType join table. This is mostly
+for legacy reasons, currently unused -- normally a ServiceResponse is attached to one
+and only one Request. 
 
-= Conventional keys:
+The architecture would allows a ServiceResponse to be tied to multiple requests, 
+perhaps to support some kind of cacheing re-use in the future. But at present, the code 
+doesn't do this, a ServiceResponse will really only be related to one request. However, a 
+ServiceResponse can be related to a single Request more than once--once per each type of 
+service response. ServiceType is really a three way join, representing a ServiceResponse, 
+attached to a particular Request, with a particular ServiceTypeValue.  
+
+= Conventional keys
 
  Absolute minimum: 
  [:display_text]   Text that will be used 
@@ -104,8 +179,8 @@ class ServiceResponse < ActiveRecord::Base
   #MatchAltEdition = 'edition'
   #MatchAltWork = 'work'
 
-  def initialize(params = nil)
-    super(params)
+  def initialize(*args)
+    super
     self.service_data = {} unless self.service_data
   end
   
@@ -122,13 +197,7 @@ class ServiceResponse < ActiveRecord::Base
   def service
     @service ||= ServiceStore.instantiate_service!( self.service_id, nil )
   end
-  
-  # Returns a hash or hash-like object with properties for the service response. 
-  def view_data
-    self.service.view_data_from_service_type(self)
-  end
-  
-  
+    
   def service_data
     # Fix weird-ass char encoding bug with AR serialize and hashes.
     # https://github.com/rails/rails/issues/6538
@@ -152,65 +221,62 @@ class ServiceResponse < ActiveRecord::Base
   
 
   def take_key_values(hash)    
-    # copy it, cause we're gonna modify it
-    hash = hash.clone
-
     hash.each_pair do |key, value|
       setter = "#{key.to_s}="      
       if ( self.respond_to?(setter))
         self.send(setter, value)
-        hash.delete(key)
+      else
+        self.service_data[key] = value
       end
     end
-    # What's left is arbitrary key/values that go in service_data
-    init_service_data(hash)
   end
 
-  def init_service_data(hash)
-    hash.each {|key, value| data_values[key] = value} if hash
-  end
 
-  def data_values    
-    # Lazy load, and store a reference. Don't worry, ruby
-    # GC handles circular references no problem. 
-    unless (@data_values_proxy)  
-      @data_values_proxy = ServiceResponseDataValues.new(self)
+  def view_data    
+    unless (@data_values)  
+      h = HashWithIndifferentAccess.new
+      ServiceResponse.built_in_fields.each do |key|
+        h[key] = self.send(key)
+      end
+      h.merge!(self.service_data.deep_dup)
+
+      # add in service_type_value
+      h[:service_type_value] = self.service_type_value_name
+
+      # Handle requested i18n translations
+      translate_simple_i18n!(h)
+
+      # Optional additional transformation provided by service?
+      if service.respond_to? :transform_view_data
+        h = service.transform_view_data(h)
+      end
+
+      # Doesn't protect modifying nested structures, but
+      # protects from some problems. 
+      h.freeze 
+      @data_values = h
     end
-    return @data_values_proxy;
+    return @data_values;
   end
+  # old name now duplicate
+  alias_method :data_values, :view_data
 
   def self.built_in_fields
     @@built_in_fields
   end
   
-
-  
-end
-
-# A proxy-like class, to provide hash-like access to all arbitrary
-# key/value pairs stored in a ServiceResponse, whether they key/value
-# is stored in an ActiveRecord attribute (#built_in_fields) or in
-# the serialized hash in the service_data attribute. 
-# Symbols passed in will be 'normalized' to strings before being used as keys.
-# So symbols and strings are interchangeable. Normally, keys should be symbols.
-class ServiceResponseDataValues
-  def initialize(arg_service_response)
-    @service_response = arg_service_response
-  end
-
-  def [](key)        
-    if ServiceResponse.built_in_fields.include?(key)
-      return @service_response.send(key)
-    else
-      return @service_response.service_data[key]
+  protected
+  # replaces :display_text and :notes with simple i18n key lookups
+  # iff :display_text_i18n and :notes_i18n are defined
+  #
+  # i18n lookups use Service.translate, to use standard scopes for
+  # this service. 
+  def translate_simple_i18n!(hash)
+    if key = hash[:display_text_i18n]
+      hash[:display_text] = self.service.translate(key, :default => hash[:display_text])
     end
-  end
-
-  def []=(key, value)
-    if(ServiceResponse.built_in_fields.include?(key))
-      @service_response.send(key.to_s+'=', value)
-    else
-      @service_response.service_data[key] = value
+    if key = self[:notes_i18n]
+      hash[:notes] = self.service.translate(key, :default => hash[:notes])
     end
   end
   

data/app/models/service_type_value.rb

@@ -4,10 +4,18 @@ require 'yaml'
 # This model is the actual list of valid service types. Not ActiveRecord,
 # just load from config files into memory (loaded on file load). 
 #
-# ServiceTypeValue's also have displable strings, stored in the
-# display_name attribute, and possibly custom display_name_plural
+# ServiceTypeValues have human-displayable names, that are controlled by Rails I18n, 
+# using standard Rails I18n pluralization forms, so for instance:
+#
+# For a ServiceTypeValue with name 'fulltext', key in i18n at 'umlaut.service_type_names.fulltext.one'
+# represents the singular (non-plural) name of objects of this ServiceTypeValue, and
+# 'umlaut.service_type_names.fulltext.other' is the plural name. 
+#
+# Other languages may have more complex pluralization rules, but Umlaut at present only
+# looks up a plural form for 10 items, so 'other' is probably all that is used. 
 class ServiceTypeValue   
-  attr_accessor :name, :id, :display_name, :display_name_plural
+  attr_accessor :name, :id
+  attr_writer :display_name, :display_name_plural
   
   class << self; attr_accessor :values; end  
   
@@ -24,11 +32,20 @@ class ServiceTypeValue
   def self.[](name)
     find(name)
   end
+
+  def display_name
+    I18n.t(self.name, :scope => "umlaut.service_type_names", :default => :default, :count => 1)
+  end
   
+  # Some languages may have multiple plural forms, but Umlaut at
+  # present is not sophisticated enough to use em all, we just look up
+  # the plural form for 10 items -- most Umlaut use of plural form
+  # is for either zero or indetermininate number. But it still may need
+  # enhancing. 
   def display_name_pluralize
-    return self.display_name_plural || self.display_name.pluralize
+    I18n.t(self.name, :scope => "umlaut.service_type_names", :default => :default, :count => 10)
   end
-  
+
   @@distro_conf_file = File.join(Umlaut::Engine.root, "db", "orig_fixed_data", "service_type_values.yml")
   @@local_conf_file = File.join(Rails.root, "config", "umlaut_service_type_values.yml")
   

data/app/models/service_wave.rb

@@ -62,11 +62,27 @@ class ServiceWave
           
           begin
             local_request = Request.connection_pool.with_connection do
-              # pre-load all relationships so no ActiveRecord activity will be
-              # needed later to see em. 
-              Request.includes(:referent, :service_responses, :dispatched_services).find(request_id)
+              # We are attempting to pre-load all relationships, both for efficiency,
+              # and so our thread can use them all without needing to checkout
+              # an ActiveRecord connection. 
+              req = Request.includes({:referent => :referent_values}, :service_responses, :dispatched_services).find(request_id)
+
+              # It turns out even though referent.referent_values is loaded from the db, on first
+              # access Rails will still access #connection, triggering a checkout. We force
+              # that to happen here, in our with_connection, so it won't happen later. 
+              #
+              # Yeah, this is a hacky mess, ActiveRecord isn't happy using it the way we are. 
+              # Should we just surround all of handle_wrapper in a with_connection checkout?
+              # Maybe. But it would require more connections in the pool to have those
+              # longer checkouts. 
+              req.referent.referent_values
+              req.service_responses
+              req.dispatched_services
+
+              req
             end
-            
+
+              
   
             if prepare_dispatch!(local_request, local_service)
               local_service.handle_wrapper(local_request)
@@ -75,7 +91,7 @@ class ServiceWave
             end
             
            
-          rescue Exception => e
+          rescue StandardError => e
             # We may not be able to access ActiveRecord because it may
             # have been an AR connection error, perhaps out of connections
             # in the pool. So log and record in non-AR ways. 
@@ -85,10 +101,15 @@ class ServiceWave
             
             # Log it too, although experience shows it may never make it to the 
             # log for mysterious reasons. 
-            Rails.logger.error(TermColor.color("Umlaut: Threaded service raised exception.", :red, true) + " Service: #{service.service_id}, #{e.class} #{e.message}. Backtrace:\n  #{e.backtrace.join("\n  ")}")
+            log_msg = TermColor.color("Umlaut: Threaded service raised exception.", :red, true) + " Service: #{service.service_id}, #{e.class} #{e.message}. Backtrace:\n  #{e.backtrace.join("\n  ")}"
+            Rails.logger.error(log_msg)
             
             # And stick it in a thread variable too
-            Thread.current[:exception] = e                      
+            Thread.current[:exception] = e    
+
+            # And try to re-raise if it's one we really don't want to swallow. 
+            # Sorry, a mess. 
+            raise e if defined?(VCR::Errors::UnhandledHTTPRequestError) && e.kind_of?(VCR::Errors::UnhandledHTTPRequestError)
           ensure
             Rails.logger.info(TermColor.color("Umlaut: Completed service #{local_service.service_id}", :yellow)+ ",  level #{@priority_level}, request #{local_request && local_request.id}: in #{Time.now - service_start}.") if @log_timing
           end
@@ -112,11 +133,11 @@ class ServiceWave
       aThread.join
 
       if aThread[:exception] 
-        debugger if Rails.env.development?
+        debugger if ENV["UMLAUT_AUTO_DEBUGGER"]
         begin
           request.dispatched(aThread[:service], DispatchedService::FailedFatal, aThread[:exception])
         rescue Exception => e
-          debugger if Rails.env.development?
+          debugger if ENV["UMLAUT_AUTO_DEBUGGER"]
           raise e
         end
       end