umlaut 3.0.0alpha13 → 3.0.0alpha14

data/Rakefile

@@ -33,5 +33,7 @@ Rake::TestTask.new(:test) do |t|
   t.verbose = false
 end
 
+require 'single_test'
+SingleTest.load_tasks
 
 task :default => :test

data/app/models/collection.rb

@@ -100,38 +100,40 @@ class Collection
     # we're going to let it keep doing it's thing in the background after
     # we return a response to the browser
     backgroundThread = Thread.new(self, umlaut_request) do | t_collection,  t_request|
+      
       # Tell our AR extension not to allow implicit checkouts
       ActiveRecord::Base.forbid_implicit_checkout_for_thread! if ActiveRecord::Base.respond_to?("forbid_implicit_checkout_for_thread!")
       
-      # got to reserve an AR connection for our main 'background traffic director'
-      # thread, so it has a connection to use to mark services as failed, at least. 
-      ActiveRecord::Base.connection_pool.with_connection do
-        begin
-          # Deal with ruby's brain dead thread scheduling by setting
-          # bg threads to a lower priority so they don't interfere with fg
-          # threads.
-          Thread.current.priority = -1          
+      begin
+        # Set priority to lower for background thread; may or may not
+        # actually have an effect in MRI, unclear, but can't hurt. 
+        prior = Thread.current.priority
+        Thread.current.priority = prior - 1          
         
-          ('a'..'z').each do | priority |
-            services_to_run = self.instantiate_services!(:level => priority, :ids => queued_service_ids)        
-            next if services_to_run.empty?      
-            ServiceWave.new(services_to_run , priority).handle(umlaut_request, umlaut_request.session_id)                               
-          end        
-       rescue Exception => e
-          # We are divorced from any HTTP request at this point, and may not
-          # have access to an ActiveRecord connection. Not much
-          # we can do except log it. 
-          # If we're catching an exception here, service processing was
-          # probably interrupted, which is bad. You should not intentionally
-          # raise exceptions to be caught here.
-          #
-          # Normally even unexpected exceptions were caught inside the ServiceWave,
-          # and logged to db as well as logfile if possible, only bugs in ServiceWave
-          # itself should wind up caught here. 
-          Thread.current[:exception] = e
-          logger.error("Background Service execution exception: #{e}\n\n   " + clean_backtrace(e).join("\n"))                
-       end
-     end
+        # Try to give the thread scheduler another hint, really, get
+        # other stuff done before this thread. 
+        Thread.pass
+        
+        
+        ('a'..'z').each do | priority |
+            services_to_run = self.instantiate_services!(:level => priority, :ids => queued_service_ids)
+          next if services_to_run.empty?
+          ServiceWave.new(services_to_run , priority).handle(umlaut_request, umlaut_request.session_id)                               
+        end        
+      rescue Exception => e
+        # We are divorced from any HTTP request at this point, and may not
+        # have access to an ActiveRecord connection. Not much
+        # we can do except log it. 
+        # If we're catching an exception here, service processing was
+        # probably interrupted, which is bad. You should not intentionally
+        # raise exceptions to be caught here.
+        #
+        # Normally even unexpected exceptions were caught inside the ServiceWave,
+        # and logged to db as well as logfile if possible, only bugs in ServiceWave
+        # itself should wind up caught here. 
+        Thread.current[:exception] = e
+        logger.error("Background Service execution exception: #{e}\n\n   " + clean_backtrace(e).join("\n"))                
+      end
     end
   end
   

data/app/models/request.rb

@@ -50,18 +50,17 @@ class Request < ActiveRecord::Base
     # up in the db to see if we've seen it before. 
     param_fingerprint = self.co_params_fingerprint( co_params )
     client_ip = params['req.ip'] || a_rails_request.remote_ip()
+    
     unless (req || params["umlaut.force_new_request"] == "true" || param_fingerprint.blank? )
       # If not found yet, then look for an existing request that had the same
       # openurl params as this one, in the same session. In which case, reuse.
       # Here we do require same session, since we don't have an explicit
       # request_id given.
-      
       req = Request.where(
                   :session_id => a_rails_request.session_options[:id],
                   :contextobj_fingerprint => param_fingerprint, 
                   :client_ip_addr => client_ip ).
-        order("created_at DESC").first
-                  
+          order("created_at DESC, id DESC").first
     end
     
     # Okay, if we found a req, it might NOT have a referent, it might
@@ -74,7 +73,6 @@ class Request < ActiveRecord::Base
       # didn't find an existing one at all, just create one
       req = self.create_new_request!( :params => params, :session => session, :rails_request => a_rails_request, :contextobj_fingerprint => param_fingerprint, :context_object => context_object )
     end
-
     return req
   end
     

data/app/models/service_wave.rb

@@ -25,7 +25,7 @@ class ServiceWave
   def prepare_dispatch!(request, service, session_id)
     can_dispatch = false
     
-    ActiveRecord::Base.connection_pool.with_connection do
+    Request.connection_pool.with_connection do
       if request.can_dispatch?( service)
         # Mark this service as in progress in the dispatch table.
         request.dispatched( service, DispatchedService::InProgress )
@@ -69,8 +69,9 @@ class ServiceWave
   
           # Tell our AR extension not to allow implicit checkouts
           ActiveRecord::Base.forbid_implicit_checkout_for_thread! if ActiveRecord::Base.respond_to?("forbid_implicit_checkout_for_thread!")
+          
           begin
-            local_request = ActiveRecord::Base.connection_pool.with_connection do
+            local_request = Request.connection_pool.with_connection do
               # pre-load all relationships so no ActiveRecord activity will be
               # needed later to see em. 
               Request.includes(:referent, :service_responses, :dispatched_services).find(request_id)
@@ -119,14 +120,9 @@ class ServiceWave
     # Wait for all the threads to complete, if any. 
     threads.each do |aThread|
       aThread.join
-      #aThread.kill # shouldn't be neccesary, but I'm paranoid
-      
-     
-      
+
       if aThread[:exception] 
         debugger
-        # mark it finished in the db, we should have a connection already, we're
-        # in main request thread. 
         begin
           request.dispatched(aThread[:service], DispatchedService::FailedFatal, aThread[:exception])
         rescue Exception => e
@@ -134,21 +130,16 @@ class ServiceWave
           raise e
         end
       end
-      
+
       # Okay, raise if exception, if desired.
       if ( aThread[:exception] && self.forward_exceptions? )        
         raise aThread[:exception]
       end
     end
-    
+
     threads.clear # more paranoia
-    
-    # Clean up any leftover connections left open by threads.
-    #ActiveRecord::Base.clear_active_connections!
-    # Sadly, we aren't allowed/ought not to call this method manually
-    # anymore, just have to hope we haven't left any unclosed. 
-    # https://github.com/rails/rails/pull/1200
-    #ActiveRecord::Base.connection_pool.clear_stale_cached_connections!
+
+
     Rails.logger.info(TermColor.color("Umlaut: Completed service wave #{@priority_level}", :yellow) + ", request #{request.id}: in #{Time.now - bundle_start}") if some_service_executed && @log_timing
   end
 

data/app/views/admin/service_errors/_dispatched_service.html.erb

@@ -2,7 +2,7 @@
   <td><%= dispatched_service.updated_at.to_s(:short) %></td>
   <td><%= dispatched_service.service_id %></td>    
   <td><%= dispatched_service.status %></td>
-  <td><%= link_to "[Re-request live]", url_for_with_co({:controller => "/resolve", :action => :index},  dispatched_service.request.to_context_object) %></td>
+  <td><%= link_to "[Re-request live]", url_for_with_co({:controller => "/resolve", :action => :index, "umlaut.force_new_request" => "true"},  dispatched_service.request.to_context_object) %></td>
   <td><%= link_to "[View cached request]", url_for(:controller => "/resolve", :action => :index, "umlaut.request_id" => dispatched_service.request_id ) %></td>
 </tr>
 <tr>

data/config/initializers/extensions/connection_pool.rb

@@ -0,0 +1,97 @@
+# NOTE:  THIS NEEDS TO BE REMOVED WHEN RAILS IS FIXED! 
+# We expect it will be fixed in rails 3.2.3 (and broken again in 4, sorry)
+# We require at least 3.2.2, so patch only if we're 3.2.2
+
+if Rails.version == "3.2.2" 
+
+
+  class ActiveRecord::ConnectionAdapters::ConnectionPool
+    
+    ##########################
+    # Monkey patch to fix disastrous concurrency bug in 
+    # ActiveRecord ConnectionPool#with_concurrency
+    #
+    # See https://github.com/rails/rails/issues/5330
+    ##########################
+    
+    # New method we're adding viz a viz rails 3.2.2
+    # Check to see if there is an active connection for
+    # current_connection_id, that is by default the current
+    # thread.
+    def current_connection?
+      @reserved_connections.has_key?(current_connection_id)
+    end
+    
+    # Redefine with_connection viz a viz rails 3.2.2 to
+    # properly check back in connections. 
+    # If a connection already exists yield it to the block. If no connection
+    # exists checkout a connection, yield it to the block, and checkin the
+    # connection when finished.
+    def with_connection
+      connection_id = current_connection_id
+      fresh_connection = true unless current_connection?
+      yield connection
+    ensure
+      release_connection(connection_id) if fresh_connection
+    end
+    
+    
+    ###################################
+    #
+    # Monkey patch to fix bug with threads waiting
+    # on connection waking up, not succesfuly getting
+    # a connection, and giving up too early. 
+    #
+    # https://github.com/rails/rails/pull/5422
+    #
+    # yeah, we got to replace all of #checkout.
+    # that's why we only apply this patch in 3.2.2, not
+    # in future rails versions. 
+    #####################################
+    def checkout
+        synchronize do
+          waited_time = 0
+
+          loop do
+            conn = @connections.find { |c| c.lease }
+
+            unless conn
+              if @connections.size < @size
+                conn = checkout_new_connection
+                conn.lease
+              end
+            end
+
+            if conn
+              checkout_and_verify conn
+              return conn
+            end
+
+            if waited_time >= @timeout
+              raise ConnectionTimeoutError, "could not obtain a database connection#{" within #{@timeout} seconds" if @timeout} (waited #{waited_time} seconds). The max pool size is currently #{@size}; consider increasing it."
+            end
+
+            # Sometimes our wait can end because a connection is available,
+            # but another thread can snatch it up first. If timeout hasn't
+            # passed but no connection is avail, looks like that happened --
+            # loop and wait again, for the time remaining on our timeout. 
+            before_wait = Time.now
+            @queue.wait( [@timeout - waited_time, 0].max )
+            waited_time += (Time.now - before_wait)
+
+            # Will go away in Rails 4, when we don't clean up
+            # after leaked connections automatically anymore. Right now, clean
+            # up after we've returned from a 'wait' if it looks like it's
+            # needed, then loop and try again. 
+            if(active_connections.size >= @connections.size)
+              clear_stale_cached_connections!
+            end
+          end
+        end
+      end
+    
+    
+    
+  end
+  
+end

data/lib/generators/umlaut/install_generator.rb

@@ -36,8 +36,9 @@ module Umlaut
 # production and development. sqlite3 has unclear semantics under threaded 
 # concurrency which umlaut uses, and in many cases simply does not work. 
 #
-# A high (connection) pool setting is also, sadly, generally
-# required at present. 
+# A higher pool size than ordinary is recommended because of umlaut's
+# use of concurrency. Perhaps as large as the number of services
+# you have configured to run in the same wave, plus another few. 
 #
 # development:
 #   adapter: mysql2
@@ -45,7 +46,7 @@ module Umlaut
 #   username:
 #   password:
 #   database:
-#   pool: 30
+#   pool: 15 
 
           eos
         end

data/lib/umlaut/version.rb

@@ -1,3 +1,3 @@
 module Umlaut
-  VERSION = "3.0.0alpha13"
+  VERSION = "3.0.0alpha14"
 end

data/test/dummy/config/umlaut_services.yml

@@ -105,7 +105,7 @@ default:
         type: GoogleBookSearch
         disabled: true
         priority: c        
-        api_key: 'YOUR_GBS_API_KEY'
+        #api_key: 'YOUR_GBS_API_KEY'
                 
       Ulrichs:
         display_name: "Ulrich's Directory"

data/test/integration/request_reuse_test.rb

@@ -1,7 +1,16 @@
 require 'test_helper'
 
 class RequestReuseTest < ActionDispatch::IntegrationTest
+  def setup
+    # for these tests, we don't want to actually execute any services, zero it out.
+    @orig_service_store_config = ServiceStore.config["default"]["services"]
+    ServiceStore.config["default"]["services"] = {}        
+  end
   
+  def teardown
+    # restore original services, see setup
+    ServiceStore.config["default"]["services"] = @orig_service_store_config
+  end
     
   test "reuse_of_request_in_session" do
     sess = open_session
@@ -33,7 +42,8 @@ class RequestReuseTest < ActionDispatch::IntegrationTest
   end
   
   test "umlaut.force_new_request" do
-    request_params = { :issn => "012345678" }
+    
+    request_params = { :issn => "012345679" }
     
     sess1 = open_session                    
     sess1.get "/resolve", request_params        
@@ -43,15 +53,14 @@ class RequestReuseTest < ActionDispatch::IntegrationTest
     sess1.get "/resolve", request_params.merge("umlaut.force_new_request" => "true")
     request_after_forced = sess1.assigns[:user_request].id
     
-    
     assert_not_equal( request_after_forced, created_request_id  )
-    
+
     # Make a request again without force param, still get the latter request,
     # the one created when we forced it.     
     sess1.get "/resolve", request_params    
     assert_equal( sess1.assigns[:user_request].id, request_after_forced  )        
   end
   
-
+  
   
 end

data/test/integration/request_test.rb

@@ -0,0 +1,25 @@
+require 'test_helper'
+
+class RequestReuseTest < ActionDispatch::IntegrationTest
+
+    
+  test "simple request" do
+    sess = open_session
+    
+    request_params = { :issn => "012345678" }
+    
+    sess.get "/resolve", request_params
+            
+    
+    # TODO: We ought to be waiting for all bg services in other
+    # threads before ending test. Tricky to figure out
+    # how to do cleanly, we don't even know the request id at the
+    # integration level, unless we do an API request instead of
+    # an HTML request. We'll just wait, sorry. 
+    puts "sleeping"
+    sleep 4
+  end
+  
+  
+  
+end

data/test/performance/browsing_test.rb

@@ -3,7 +3,7 @@ require 'rails/performance_test_help'
 
 # Profiling results for each test method are written to tmp/performance.
 class BrowsingTest < ActionDispatch::PerformanceTest
-  def test_homepage
-    get '/'
-  end
+  #def test_homepage
+  #  get '/'
+  #end
 end

data/test/test_helper.rb

@@ -11,6 +11,9 @@ Rails.backtrace_cleaner.remove_silencers!
 # Load support files
 Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
 
+# Complete stack trace with deprecation warnings from rails
+ActiveSupport::Deprecation.debug = true
+
 # Custom method for NYU-only tests. We still have a bunch of tests for aleph/
 # primo that work against live nyu services and can only succeed if you are 
 # nyu. Oops. We provide this convenience function to wrap test_* class
@@ -44,3 +47,4 @@ def nyu_only_tests(test_name="Some")
     yield
   end
 end
+