umlaut 3.0.0alpha1

data/lib/tasks/umlaut.rake

@@ -0,0 +1,134 @@
+
+namespace :umlaut do
+    desc "Perform nightly maintenance. Set up in cron."
+    task :nightly_maintenance => [:load_sfx_urls, :expire_sessions, :expire_old_data]
+  
+  
+
+    desc "Loads sfx_urls from SFX installation. SFX mysql login needs to be set in config."
+    task :load_sfx_urls => :environment do
+
+      if SfxDb.connection_configured?
+    
+        puts "Loading SFXUrls via direct access to SFX db."
+        #sfxlcl41.TARGET_SERVICE
+        # Check if we have an SFX3 schema, or if not use SFX4
+        sfx3 = true
+        begin
+          SfxDb::Object.connection.select_all("SHOW FIELDS FROM TARGET_SERVICE")
+        rescue ActiveRecord::StatementInvalid
+          sfx3 = false
+        end
+        
+        if sfx3
+          urls = SfxDb::SfxDbBase.fetch_sfx_urls
+        else
+          urls = SearchMethods::Sfx4.fetch_sfx_urls
+        end
+        
+        ignore_urls = UmlautController.umlaut_config.lookup!("sfx.sfx_load_ignore_hosts", [])
+        
+        # We only want the hostnames
+        hosts = urls.collect do |u|
+          begin
+          uri = URI.parse(u)
+          uri.host
+          rescue Exception
+          end
+        end
+        hosts.uniq!
+        
+        SfxUrl.transaction do
+          SfxUrl.delete_all
+          
+          hosts.each {|h| SfxUrl.new({:url => h}).save! unless ignore_urls.find {|ignore| ignore === h }}      
+        end
+      else
+        puts "Skipping load of SFXURLs via direct access to SFX db. No direct access is configured. Configure in config/umlaut_config/database.yml"
+      end
+    end
+
+    desc "Expire sessions older than config.app_config.session_expire_seconds"
+    task :expire_sessions => :environment do
+      # Assume sessions are in db. 
+      # Don't know good way to get the connection associated with sessions,
+      # since there is no model. Assume Request is in the same db.
+      expire_seconds = UmlautController.umlaut_config.lookup!("session_expire_seconds", 1.day)
+      puts "Expiring sessions older than #{expire_seconds} seconds (set with config session_expire_seconds)."
+      Request.connection.execute("delete from sessions where now() - updated_at > #{expire_seconds}")
+    end
+
+
+    desc "Cleanup of database for old data associated with expired sessions etc."
+    task :expire_old_data => :environment do
+      # There are requests, responses, and dispatched_service entries
+      # hanging around for things that may be way old and no longer
+      # need to hang around. How do we know if they're too old?
+      # If they are no longer associated with any session, mainly.
+
+      # Deleting things as aggressively as we're doing here doesn't leave
+      # us much for statistics, but we aren't currently gathering any
+      # statistics anyway. If statistics are needed, more exploration
+      # is needed of performance vs. leaving things around for statistics. 
+
+      # For efficiency, we delete with direct DB calls, so don't count
+      # on Rails business logic being triggered! Was just WAY too slow
+      # otherwise. Also, sorry, doing all this in a db efficient way (one db
+      # query) requires some tricky SQL, which be MySQL specific. 
+
+      # Current Umlaut never re-uses a request different between sessions, so
+      # if the session is dead, we can purge the Requests too. Permalink
+      # architecture has been fixed to not rely on requests or referents,
+      # permalinks (post new architecture) store their own context object.
+
+      puts "Deleting Requests no longer associated with a session."
+      begin_time = Time.now
+      work_clause = " FROM requests LEFT OUTER JOIN sessions ON requests.session_id = sessions.session_id WHERE sessions.id is null "
+      count = Request.count_by_sql("SELECT count(*) " + work_clause)
+      Request.connection.execute("DELETE requests " + work_clause)
+      puts "  Deleted #{count} Requests in #{Time.now - begin_time}"
+
+    
+
+
+      
+      # Now, let's get rid of any ServiceResponses that no longer have
+      # Requests 
+       
+      puts "Deleting orphaned ServiceResponses...."
+      begin_time = Time.now
+        work_clause = " FROM service_responses WHERE NOT EXISTS (SELECT * FROM requests WHERE service_responses.request_id =  requests.id)"
+      count = ServiceResponse.count_by_sql("SELECT count(*) " + work_clause)
+      ServiceResponse.connection.execute("DELETE " + work_clause)  
+      puts "  Deleted #{count} ServiceResponses in #{Time.now - begin_time}"
+
+      
+      # And get rid of DispatchedServices for 'dead' requests too. Don't
+      # need em.
+      puts "Deleting DispatchedServices for dead Requests..."
+      begin_time = Time.now
+      # Sorry, may be MySQL only. 
+      work_clause = " FROM (dispatched_services LEFT OUTER JOIN requests ON dispatched_services.request_id = requests.id)  WHERE requests.id IS NULL  "
+      count = DispatchedService.count_by_sql("SELECT count(*) " + work_clause)
+      DispatchedService.connection.execute("DELETE dispatched_services " + work_clause)
+      puts "  Deleted #{count} DispatchedServices in #{Time.now - begin_time}"
+      
+
+      # Turns out we need to get rid of old referents and referentvalues
+      # too. There are just too many. Permalinks have been updated to
+      # store their own info and not depend on Referent existing. 
+      referent_expire = Time.now - UmlautController.umlaut_config.lookup!("referent_expire_seconds", 20.days)
+      puts "Deleting Referents/ReferentValues older than #{referent_expire.inspect}"
+      begin_time = Time.now
+      # May be MySQL dependent. 
+      Referent.connection.execute("DELETE referents, referent_values FROM referents, referent_values where referents.id = referent_values.referent_id AND referents.created_at < '#{referent_expire.to_formatted_s(:db)}'" )
+      puts "  Deleted Referents in #{Time.now - begin_time}"
+      
+      # And turns out we have all Clickthroughs being stored for no apparent
+      # reason, let's just delete any older than 3 months ago. 
+      Clickthrough.destroy_all(['created_at < ?', 3.months.ago])
+      puts "Deleted Clickthroughs older than 3 months"
+              
+    end
+    
+end

data/lib/umlaut/default_configuration.rb

@@ -0,0 +1,5 @@
+module Umlaut::DefaultConfiguration
+  
+  
+  
+end

data/lib/umlaut/routes.rb

@@ -0,0 +1,136 @@
+# -*- encoding : utf-8 -*-
+module Umlaut
+  # Class to inject Umlaut routes, design copied from Blacklight project. 
+  # you would do a:
+  #    Umlaut::Routes.new(self, optional_args).draw 
+  # in local
+  # app routes.rb, that line is generated into local app by Umlaut generator.
+  # options include :only and :except to limit what route groups are generated. 
+  class Routes
+
+    def initialize(router, options ={})
+      @router = router
+      @options = options
+    end
+
+    def draw
+      route_sets.each do |r|
+        self.send(r)
+      end
+    end
+
+    protected
+
+    def add_routes &blk
+      @router.instance_exec(@options, &blk)
+    end
+
+    def route_sets
+      (@options[:only] || default_route_sets) - (@options[:except] || [])
+    end
+
+    def default_route_sets
+      [:root, :permalinks, :a_z, :resolve, :open_search, :link_router, :export_email, :resources, :search]
+    end
+
+    module RouteSets
+      # for now include root generation in Umlaut auto-generation
+      def root
+        add_routes do |options|
+          root :to => "resolve#index"
+        end
+      end
+      
+      def permalinks
+        add_routes do |options|
+          match 'go/:id' => 'store#index'
+        end
+      end
+  
+      # some special direct links to A-Z type searches, including
+      # legacy redirects for SFX-style urls, to catch any bookmarks. 
+      def a_z
+        add_routes do |options|
+          # Special one for alpha list
+          match 'journal_list/:id/:page' => 'search#journal_list', :defaults => { :page => 1, :id => 'A' }
+          
+          
+          # Catch redirected from SFX A-Z and citation linker urls
+          # v2 A-Z links redirected to umlaut, point to journal_list
+          # code in journal_list filter picks out SFX URL vars for
+          # letter. 
+          match '/resolve/azlist/default' => 'search#journal_list', :page => 1, :id => 'A'
+          
+          # SFX v3 A-Z list url format
+          match 'resolve/az' => 'search#journal_list', :page => 1, :id => 'A'          
+        end
+      end
+      
+        # This is a legacy wild controller route that's not recommended for RESTful applications.
+        # Note: This route will make all actions in every controller accessible via GET requests.
+        # match ':controller(/:action(/:id(.:format)))'
+        
+      def resolve
+        add_routes do |options|
+          # ResolveController still uses rails 2.0 style 'wildcard' routes, 
+          # TODO tighten this up to only match what oughta be matched.           
+          # Note: This route will make all actions in this controller accessible via GET requests.
+          
+          match 'resolve(/:action(/:id(.:format)))' => "resolve"
+        end
+      end
+      
+      def open_search
+        add_routes do |options|
+          # OpenSearchController still uses rails 2.0 style 'wildcard' routes, 
+          # TODO tighten this up to only match what oughta be matched.           
+          # Note: This route will make all actions in this controller accessible via GET requests.
+          
+          match 'open_search(/:action(/:id(.:format)))' => "open_search"
+        end
+      end
+      
+      def link_router
+        add_routes do |options|
+          # LinkRouterController still uses rails 2.0 style 'wildcard' routes, 
+          # TODO tighten this up to only match what oughta be matched.           
+          # Note: This route will make all actions in this controller accessible via GET requests.
+          
+          match 'link_router(/:action(/:id(.:format)))' => "link_router"
+        end
+      end
+      
+      def export_email
+        add_routes do |options|
+          # ExportEmailController still uses rails 2.0 style 'wildcard' routes, 
+          # TODO tighten this up to only match what oughta be matched.           
+          # Note: This route will make all actions in this controller accessible via GET requests.
+          
+          match 'export_email(/:action(/:id(.:format)))' => "export_email"
+        end
+      end
+      
+      def resources
+        add_routes do |options|
+          # ResourceController still uses rails 2.0 style 'wildcard' routes, 
+          # TODO tighten this up to only match what oughta be matched.           
+          # Note: This route will make all actions in this controller accessible via GET requests.
+          
+          match 'resource(/:action(/:id(.:format)))' => "resource"
+        end
+      end
+      
+      def search
+        add_routes do |options|
+          # SearchController still uses rails 2.0 style 'wildcard' routes, 
+          # TODO tighten this up to only match what oughta be matched.           
+          # Note: This route will make all actions in this controller accessible via GET requests.
+          
+          match 'search(/:action(/:id(.:format)))' => "search"
+        end
+      end
+   
+    end
+    include RouteSets
+  end
+end

data/lib/umlaut/version.rb

@@ -0,0 +1,3 @@
+module Umlaut
+  VERSION = "3.0.0alpha1"
+end

data/lib/umlaut.rb

@@ -0,0 +1,37 @@
+require 'umlaut/routes'
+
+# not sure why including openurl gem doesn't do the require, but it
+# seems to need this. 
+require 'openurl'
+
+module Umlaut
+  class Engine < Rails::Engine
+    engine_name "umlaut"
+    
+    # we store some things in non-standard subdirs, add em to path.
+    #
+    # We EAGER load em all to try and handle threading issues. 
+    
+    config.autoload_paths << File.join(self.root, "lib", "referent_filters")
+    config.eager_load_paths << File.join(self.root, "lib", "referent_filters")
+    
+    config.autoload_paths << File.join(self.root, "lib", "service_adaptors")
+    config.eager_load_paths << File.join(self.root, "lib", "service_adaptors")
+    
+    # Ane make 'lib' dir auto-loaded, because we have legacy Rails2 code
+    # that assumes it.
+    config.autoload_paths << File.join(self.root, "lib")
+    # Sadly including eager_load_paths here makes weird things happen, apparently
+    # I don't entirely understand what's going on. 
+    #config.eager_load_paths << File.join(self.root, "lib")
+    
+    # This makes our rake tasks visible.
+    rake_tasks do
+      Dir.chdir(File.expand_path(File.join(File.dirname(__FILE__), '..'))) do
+        Dir.glob(File.join('lib', 'tasks', '*.rake')).each do |railtie|
+          load railtie
+        end
+      end
+    end
+  end
+end

data/lib/umlaut_configurable.rb

@@ -0,0 +1,343 @@
+# meant to be included in _controllers_, to get an
+# umlaut_config method as a class_attribute (avail on class, overrideable
+# on instance), exposed as helper method too, 
+# that has a Confstruct configuration object that starts out
+# holding global config. (right now via a direct refernce to the global
+# one). 
+
+require 'confstruct'
+
+module UmlautConfigurable
+  extend ActiveSupport::Concern
+  
+  included do
+    class_attribute :umlaut_config
+    helper_method :umlaut_config
+    self.umlaut_config = Confstruct::Configuration.new
+  end
+   
+  
+  
+  # Call as UmlautConfigurable.set_default_configuration!(confstruct_obj)
+  # to initialize
+  def self.set_default_configuration!(configuration)
+    configuration.configure do
+      app_name 'Find It'
+      # URL to image to use for link resolver, OR name of image asset in local app. 
+      #link_img_url "http//something"
+      
+      # Sometimes Umlaut sends out email, what email addr should it be from?
+      from_email_addr 'no_reply@umlaut.example.com'
+          
+      layout "umlaut"
+      resolve_layout deferred! {|c| c.layout}    
+      search_layout deferred! {|c| c.layout}
+      
+      # help url used on error page and a few other places.
+      # help_url  "http://www.library.jhu.edu/services/askalib/index.html"
+      
+      # Minimum height and width of browser window. We have little control over
+      # what size a content provider generates a window for a link resolver. Often
+      # it's too small for umlaut. So we resize in js, if these config params
+      # are given. Set to 0 to disable.
+      # Sadly, only some browsers let us resize the browser window, so this
+      # feature only works in some browsers. 
+      minimum_window_width    820
+      minimum_window_height   400
+      
+      
+      # rfr_ids used for umlaut generated pages.
+      rfr_ids do
+        opensearch  "info:sid/umlaut.code4lib.org:opensearch"
+        citation    "info:sid/umlaut.code4lib.org:citation"
+        azlist      'info:sid/umlaut.code4lib.org:azlist'
+      end
+      
+      # If you have a test umlaut set up at another location to stage/test
+      # new features, link to it here, and a helper method in default
+      # layout will provide a subtle debugging link to it in footer,
+      # for current OpenURL. 
+      # test_resolve_base "http://app01.mse.jhu.edu/umlaut_dev"
+      
+      opensearch_short_name deferred! {|c| "Find Journals with #{c.app_name}" }
+      opensearch_description deferred! {|c| "Search #{c.app_name} for journal names containing your term"}
+      
+      
+      
+      # Referent filters. Sort of like SFX source parsers.
+      # hash, key is regexp to match a sid, value is filter object
+      # (see lib/referent_filters )        
+      add_referent_filters!( :match => /.*/, :filter => DissertationCatch.new ) 
+      
+        
+      # skip_resolve_menu can be used to control 'direct' linking, skipping
+      # the resolve menu to deliver a full text link or other resource
+      # directly to the user.
+      # Possible values:
+      # false  : [default] Never skip menu
+      #   A hash with one or more keys....
+      # {:service_types => ['fulltext']} : list of service type values, if 
+      #          they're present skip the menu with the first response available.
+      # {:excluded_services => ['JH_HIP'] : list of service IDs, exclude responses
+      #          from these services for direct linking. (Not yet implemented)
+      # {:excluded_urls => [/regexp/, 'string'] : list of regexps or strings,
+      #          exclude URLs that match this string from being skipped to. (Not yet implemented)
+      # {:excluded_rfr_ids => ["info:sid/sfxit.com:citation", '"info:sid/umlaut.code4lib.org:citation"'] }
+      # {:lambda => lambda {|p, l| return something}} : Not yet implemented. 
+      
+      # lambda expression: A lambda expression can be provided that
+      #          should expect one argument, a hash with key :request
+      #          and value the Umlaut Request object. Return nil to
+      #          not skip menu, or a ServiceType join obj to skip
+      #          menu to that response.
+    
+      # A pretty typical direct-linking setup, excludes queries that come
+      # from citation linker/azlist/opensearch from direct linking. 
+      # AppConfig::Base.skip_resolve_menu = {:service_types => ['fulltext'],
+      #:services=>['JH_SFX'], :excluded_rfr_ids => ["info:sid/sfxit.com:citation",
+      #'info:sid/umlaut.code4lib.org:citation',
+      #'info:sid/umlaut.code4lib.org:azlist',  
+      #'info:sid/umlaut.code4lib.org:opensearch']}
+      #
+      # "umlaut.skip_resolve_menu" paramter can also be passed in per-request, with
+      # 'true' or shortname of a service type. 
+      skip_resolve_menu false
+      
+      # How many seconds between updates of the background updater for background
+      # services?
+      poll_wait_seconds 4
+      
+      # if a background service hasn't returned in this many seconds, consider
+      # it failed. (May actually be slow, more likely raised an exception and
+      # our exception handling failed to note it as failed.)    
+      background_service_timeout 30
+      # If a service has status FailedTemporary, and it's older than a
+      # certain value, it will be re-queued in #serviceDispatch.
+      # This value defaults to 10 times background_service_timeout,
+      # but can be set in app config variable requeue_failedtemporary_services
+      # If you set it too low, you can wind up with a request that never completes,
+      # as it constantly re-queues a service which constantly fails.
+      requeue_failedtemporary_services_in deferred! {|c| c.background_service_timeout * 10}
+  
+      # custom view template for resolve#index
+      resolve_view nil
+      
+      # If OpenURL came from manual entry of title/ISSN, and no match is found in
+      # link resolver knowledge base, display a warning to the user of potential
+      # typo?
+      entry_not_in_kb_warning true
+          
+      nightly_maintenance do
+         # When nightly_maintenance will expire sessions. Default to
+         # 1 day. Over-ride locally if desired, but
+         # probably no reason to.
+        session_expire_seconds  1.day           
+        referent_expire_seconds deferred! {|c| c.session_expire_seconds }
+        
+        
+        # Expire service responses. Service responses are only re-used by the same
+        # session that generated them. But sometimes even that's too much, we
+        # want to expire them eventually, say every 24 hours. 
+        # You can do this in two ways.
+        #
+        # A Number of seconds in an interval, eg:
+        response_expire_interval  1.day
+        #
+        # Or, sometimes it's convenient to synchronize this with some other
+        # process that runs on crontab. Say, expire at midnight every night:
+        # response_expire_crontab_format  "00 00 * * *"
+  
+      end
+      
+      # Configuration for the 'search' functions -- A-Z lookup
+      # and citation entry. 
+      search do
+        # Is your SFX database connection, defined in database.yml under
+        # sfx_db and used for A-Z searches, Sfx3 or Sfx4?  Other SearchMethods
+        # in addition to SFX direct db may be provided later. 
+        az_search_method  SearchMethods::Sfx4
+        #az_search_method  SearchMethods::Sfx3
+        
+        # When talking directly to the SFX A-Z list database, you may
+        # need to set this, if you have multiple A-Z profiles configured
+        # and don't want to use the 'default.
+        sfx_az_profile "default"    
+        
+        # Use your own custom search view? mention it here.   
+        #search_view = "my_search"
+        
+        # can set to "_blank" etc. 
+        result_link_target nil
+        
+      end
+      
+      # config only relevant to SFX use  
+      sfx do
+        # was: 'main_sfx_base_url'
+        # base sfx url to use for search actions, error condition backup,
+        # and some other purposes. For search actions (A-Z), direct database
+        # connection to your SFX db also needs to be defined in database.yml
+        # sfx_base_url  'http://sfx.library.jhu.edu:8000/jhu_sfx?'
+        # 
+  
+        
+        
+        # Umlaut tries to figure out from the SFX knowledge base
+        # which hosts are "SFX controlled", to avoid duplicating SFX
+        # urls with urls from catalog. But sometimes it misses some, or
+        # alternate hostnames for some. Regexps matching against
+        # urls can be included here. Eg,
+        # AppConfig::Base.additional_sfx_controlled_urls = [
+        #    %r{^http://([^\.]\.)*pubmedcentral\.com}
+        #  ]    
+        additional_sfx_controlled_urls = []
+        
+        # "web.archive.org" is listed in SFX, but that causes suppression
+        # of MARC 856 tags from our catalog pointing to archive.org, which are
+        # being used for some digitized books. We'd like to ignore that archive.org
+        # is in SFX. Same for netlibrary.
+        #sfx_load_ignore_hosts = [/.*\.archive\.org/, /www\.netlibrary\.com/, 'www.loc.gov']
+        sfx_load_ignore_hosts = []        
+      end
+    
+      # Output timing of service execution to logs
+      log_service_timing = true if Rails.env == "development"
+      
+      
+      #####
+      # Pieces of content on a Resolve page can be declaritively configured. 
+      # Here are the defaults. You can add new elements to the resolve_sections
+      # array in config and modify or delete existing resolve_sections elements.
+      #
+      # Look in comments at top of SectionRenderer class for what the keys
+      # in each entry mean. 
+      add_resolve_sections! do
+        div_id "cover_image"
+        partial "cover_image"
+        visibility :responses_exist
+        show_heading false
+        show_spinner false
+      end
+      
+      add_resolve_sections! do
+        div_id "search_inside"
+        html_area :resource_info
+        partial "search_inside"
+        show_partial_only true
+      end
+      
+      add_resolve_sections! do
+        div_id "fulltext"
+        section_title "#{ServiceTypeValue[:fulltext].display_name} via:"
+        html_area :main
+        partial :fulltext
+        show_partial_only true
+      end
+      
+      add_resolve_sections! do
+        div_id "excerpts"
+        section_prompt "A limited preview which may include table of contents, index, and other selected pages."
+        html_area :main
+        list_visible_limit 5
+        visibility :responses_exist
+      end
+      
+      add_resolve_sections! do
+        div_id "audio"
+        section_title "#{ServiceTypeValue[:audio].display_name} via"
+        html_area :main
+        visibility :responses_exist
+      end
+      
+      add_resolve_sections! do
+        div_id :holding
+        section_title ServiceTypeValue[:holding].display_name_pluralize
+        html_area :main
+        partial 'holding'
+        service_type_values ["holding","holding_search"]
+      end
+      
+      add_resolve_sections! do
+        div_id "document_delivery"
+        section_title "Request a copy from Inter-Library Loan"
+        html_area :main
+        visibility :responses_exist
+        bg_update false
+      end
+      
+      add_resolve_sections! do
+        div_id "table_of_contents"
+        html_area :main
+        visibility :responses_exist
+      end
+      
+      add_resolve_sections! do
+        div_id "abstract"
+        html_area :main
+        visibility :responses_exist
+      end
+      
+      add_resolve_sections! do
+        div_id "help"
+        html_area :sidebar
+        bg_update false
+        partial "help"
+        show_heading false
+        show_spinner false
+        visibility :responses_exist 
+      end
+      
+      add_resolve_sections! do
+        div_id "coins"
+        html_area :sidebar
+        partial "coins"
+        service_type_values []
+        show_heading false
+        show_spinner false
+        bg_update false
+        partial_html_api false
+      end
+      
+      add_resolve_sections! do
+        div_id "export_citation"
+        html_area :sidebar
+        visibility :in_progress
+        item_name_plural "Export tools"
+      end
+      
+      add_resolve_sections! do
+        div_id "related_items"
+        html_area :sidebar
+        partial "related_items"
+        section_title "More like this"
+        item_name_plural "Related Items"
+        # custom visibility, show it for item-level cites,
+        # or if we actually have some
+        visibility(  lambda do |renderer|
+                        (! renderer.request.title_level_citation?) ||
+                        (! renderer.responses_empty?)
+                      end )
+        service_type_values ['cited_by', 'similar']
+      end
+      
+      add_resolve_sections! do
+        div_id "highlighted_link"
+        section_title "See also"
+        html_area :sidebar
+        visibility :in_progress
+        partial_locals( :show_source => true )
+      end
+      
+      add_resolve_sections! do
+        div_id "service_errors"
+        partial "service_errors"
+        html_area :service_errors
+        service_type_values []
+      end
+      
+    end
+  end
+  
+  
+  
+end