umlaut 3.0.0beta9 → 3.0.0beta10

data/active_record_patch/connection_pool.rb

@@ -2,12 +2,14 @@
 #
 #  ActiveRecord's ConnectionPool in Rails 3.2.3 allows threads to 'steal'
 #  connections from each other, so some threads get starved out. 
+#
 #  This monkey patch uses an implementation from https://github.com/rails/rails/pull/6492
-#  that ensures 'fair' queue in ConnectionPool. 
+#  that ensures 'fair' queue in ConnectionPool.
+#
+#  It's actually a weird hybrid which ALSO maintains the clear_stale_cached_connections!
+#  behavior to reclaim leaked orphaned connections, and calls that method
+#  in checkout when pool has no avail connections.
 #
-#  Can be removed if/when we are on an AR that incorporates above patch
-#  or equivalent. 
-# 
 #  This file referenced from an initializer in our main engine
 #  class, that loads it to monkey patch AR.
 #
@@ -20,6 +22,8 @@ unless ENV["NO_AR_PATCH"]
   require 'active_record'
   ActiveRecord::ConnectionAdapters::ConnectionPool 
   
+  # Unload it so we can redefine it completely
+  ActiveRecord::ConnectionAdapters.send(:remove_const, :ConnectionPool)
   
   # Some require's our new definition will need
   require 'thread'
@@ -27,427 +31,436 @@ unless ENV["NO_AR_PATCH"]
   require 'set'
   require 'active_support/core_ext/module/deprecation'
   
-  
-  
-  # And monkey patch
-  module ActiveRecord
-  
-  
-    module ConnectionAdapters
-      # Connection pool base class for managing Active Record database
-      # connections.
-      #
-      # == Introduction
-      #
-      # A connection pool synchronizes thread access to a limited number of
-      # database connections. The basic idea is that each thread checks out a
-      # database connection from the pool, uses that connection, and checks the
-      # connection back in. ConnectionPool is completely thread-safe, and will
-      # ensure that a connection cannot be used by two threads at the same time,
-      # as long as ConnectionPool's contract is correctly followed. It will also
-      # handle cases in which there are more threads than connections: if all
-      # connections have been checked out, and a thread tries to checkout a
-      # connection anyway, then ConnectionPool will wait until some other thread
-      # has checked in a connection.
-      #
-      # == Obtaining (checking out) a connection
-      #
-      # Connections can be obtained and used from a connection pool in several
-      # ways:
-      #
-      # 1. Simply use ActiveRecord::Base.connection as with Active Record 2.1 and
-      #    earlier (pre-connection-pooling). Eventually, when you're done with
-      #    the connection(s) and wish it to be returned to the pool, you call
-      #    ActiveRecord::Base.clear_active_connections!. This will be the
-      #    default behavior for Active Record when used in conjunction with
-      #    Action Pack's request handling cycle.
-      # 2. Manually check out a connection from the pool with
-      #    ActiveRecord::Base.connection_pool.checkout. You are responsible for
-      #    returning this connection to the pool when finished by calling
-      #    ActiveRecord::Base.connection_pool.checkin(connection).
-      # 3. Use ActiveRecord::Base.connection_pool.with_connection(&block), which
-      #    obtains a connection, yields it as the sole argument to the block,
-      #    and returns it to the pool after the block completes.
-      #
-      # Connections in the pool are actually AbstractAdapter objects (or objects
-      # compatible with AbstractAdapter's interface).
-      #
-      # == Options
-      #
-      # There are several connection-pooling-related options that you can add to
-      # your database connection configuration:
+  # And completely redefine ConnectionPool
+
+  class ActiveRecord::ConnectionAdapters::ConnectionPool
+      # Threadsafe, fair, FIFO queue.  Meant to be used by ConnectionPool
+      # with which it shares a Monitor.  But could be a generic Queue.
       #
-      # * +pool+: number indicating size of connection pool (default 5)
-      # * +checkout_timeout+: number of seconds to block and wait for a connection
-      #   before giving up and raising a timeout error (default 5 seconds).
-      # * +reaping_frequency+: frequency in seconds to periodically run the
-      #   Reaper, which attempts to find and close dead connections, which can
-      #   occur if a programmer forgets to close a connection at the end of a
-      #   thread or a thread dies unexpectedly. (Default nil, which means don't
-      #   run the Reaper).
-      # * +dead_connection_timeout+: number of seconds from last checkout
-      #   after which the Reaper will consider a connection reapable. (default
-      #   5 seconds).
-      class ConnectionPool
-        # Threadsafe, fair, FIFO queue.  Meant to be used by ConnectionPool
-        # with which it shares a Monitor.  But could be a generic Queue.
-        #
-        # The Queue in stdlib's 'thread' could replace this class except
-        # stdlib's doesn't support waiting with a timeout.
-        class Queue
-          def initialize(lock = Monitor.new)
-            @lock = lock
-            @cond = @lock.new_cond
-            @num_waiting = 0
-            @queue = []
-          end
-  
-          # Test if any threads are currently waiting on the queue.
-          def any_waiting?
-            synchronize do
-              @num_waiting > 0
-            end
-          end
-  
-          # Return the number of threads currently waiting on this
-          # queue.
-          def num_waiting
-            synchronize do
-              @num_waiting
-            end
-          end
-  
-          # Add +element+ to the queue.  Never blocks.
-          def add(element)
-            synchronize do
-              @queue.push element
-              @cond.signal
-            end
-          end
-  
-          # If +element+ is in the queue, remove and return it, or nil.
-          def delete(element)
-            synchronize do
-              @queue.delete(element)
-            end
-          end
-  
-          # Remove all elements from the queue.
-          def clear
-            synchronize do
-              @queue.clear
-            end
-          end
-  
-          # Remove the head of the queue.
-          #
-          # If +timeout+ is not given, remove and return the head the
-          # queue if the number of available elements is strictly
-          # greater than the number of threads currently waiting (that
-          # is, don't jump ahead in line).  Otherwise, return nil.
-          #
-          # If +timeout+ is given, block if it there is no element
-          # available, waiting up to +timeout+ seconds for an element to
-          # become available.
-          #
-          # Raises:
-          # - ConnectionTimeoutError if +timeout+ is given and no element
-          # becomes available after +timeout+ seconds,
-          def poll(timeout = nil)
-            synchronize do
-              if timeout
-                no_wait_poll || wait_poll(timeout)
-              else
-                no_wait_poll
-              end
-            end
-          end
-  
-          private
-  
-          def synchronize(&block)
-            @lock.synchronize(&block)
-          end
-  
-          # Test if the queue currently contains any elements.
-          def any?
-            !@queue.empty?
-          end
-  
-          # A thread can remove an element from the queue without
-          # waiting if an only if the number of currently available
-          # connections is strictly greater than the number of waiting
-          # threads.
-          def can_remove_no_wait?
-            @queue.size > @num_waiting
-          end
-  
-          # Removes and returns the head of the queue if possible, or nil.
-          def remove
-            @queue.shift
-          end
-  
-          # Remove and return the head the queue if the number of
-          # available elements is strictly greater than the number of
-          # threads currently waiting.  Otherwise, return nil.
-          def no_wait_poll
-            remove if can_remove_no_wait?
-          end
-  
-          # Waits on the queue up to +timeout+ seconds, then removes and
-          # returns the head of the queue.
-          def wait_poll(timeout)
-            @num_waiting += 1
-  
-            t0 = Time.now
-            elapsed = 0
-            loop do
-              @cond.wait(timeout - elapsed)
-  
-              return remove if any?
-  
-              elapsed = Time.now - t0
-              raise ConnectionTimeoutError if elapsed >= timeout
-            end
-          ensure
-            @num_waiting -= 1
-          end
-        end
-      
-  
-        include MonitorMixin
-  
-        attr_accessor :automatic_reconnect, :checkout_timeout, :dead_connection_timeout
-        attr_reader :spec, :connections, :size, :reaper
-  
-        # Creates a new ConnectionPool object. +spec+ is a ConnectionSpecification
-        # object which describes database connection information (e.g. adapter,
-        # host name, username, password, etc), as well as the maximum size for
-        # this ConnectionPool.
-        #
-        # The default ConnectionPool maximum size is 5.
-        def initialize(spec)
-          super()
-  
-          @spec = spec
-  
-          # The cache of reserved connections mapped to threads
-          @reserved_connections = {}
-  
-          @checkout_timeout = spec.config[:checkout_timeout] || 5
-          
-  
-          # default max pool size to 5
-          @size = (spec.config[:pool] && spec.config[:pool].to_i) || 5
-  
-          @connections         = []
-          @automatic_reconnect = true
-  
-          @available = Queue.new self
+      # The Queue in stdlib's 'thread' could replace this class except
+      # stdlib's doesn't support waiting with a timeout.
+      class Queue
+        def initialize(lock = Monitor.new)
+          @lock = lock
+          @cond = @lock.new_cond
+          @num_waiting = 0
+          @queue = []
         end
-  
-  
-        # Retrieve the connection associated with the current thread, or call
-        # #checkout to obtain one if necessary.
-        #
-        # #connection can be called any number of times; the connection is
-        # held in a hash keyed by the thread id.
-        def connection
+
+        # Test if any threads are currently waiting on the queue.
+        def any_waiting?
           synchronize do
-            @reserved_connections[current_connection_id] ||= checkout
+            @num_waiting > 0
           end
         end
-  
-        # Is there an open connection that is being used for the current thread?
-        def active_connection?
+
+        # Return the number of threads currently waiting on this
+        # queue.
+        def num_waiting
           synchronize do
-            @reserved_connections.fetch(current_connection_id) {
-              return false
-            }.in_use?
+            @num_waiting
           end
         end
-  
-        # Signal that the thread is finished with the current connection.
-        # #release_connection releases the connection-thread association
-        # and returns the connection to the pool.
-        def release_connection(with_id = current_connection_id)
+
+        # Add +element+ to the queue.  Never blocks.
+        def add(element)
           synchronize do
-            conn = @reserved_connections.delete(with_id)
-            checkin conn if conn
+            @queue.push element
+            @cond.signal
           end
         end
-  
-        # If a connection already exists yield it to the block. If no connection
-        # exists checkout a connection, yield it to the block, and checkin the
-        # connection when finished.
-        def with_connection
-          connection_id = current_connection_id
-          fresh_connection = true unless active_connection?
-          yield connection
-        ensure
-          release_connection(connection_id) if fresh_connection
-        end
-  
-        # Returns true if a connection has already been opened.
-        def connected?
-          synchronize { @connections.any? }
-        end
-  
-        # Disconnects all connections in the pool, and clears the pool.
-        def disconnect!
+
+        # If +element+ is in the queue, remove and return it, or nil.
+        def delete(element)
           synchronize do
-            @reserved_connections = {}
-            @connections.each do |conn|
-              checkin conn
-              conn.disconnect!
-            end
-            @connections = []
-            @available.clear
+            @queue.delete(element)
           end
         end
-  
-        # Clears the cache which maps classes.
-        def clear_reloadable_connections!
+
+        # Remove all elements from the queue.
+        def clear
           synchronize do
-            @reserved_connections = {}
-            @connections.each do |conn|
-              checkin conn
-              conn.disconnect! if conn.requires_reloading?
-            end
-            @connections.delete_if do |conn|
-              conn.requires_reloading?
-            end
-            @available.clear
-            @connections.each do |conn|
-              @available.add conn
-            end
+            @queue.clear
           end
         end
-  
-       
-        # Check-out a database connection from the pool, indicating that you want
-        # to use it. You should call #checkin when you no longer need this.
-        #
-        # This is done by either returning and leasing existing connection, or by
-        # creating a new connection and leasing it.
+
+        # Remove the head of the queue.
         #
-        # If all connections are leased and the pool is at capacity (meaning the
-        # number of currently leased connections is greater than or equal to the
-        # size limit set), an ActiveRecord::PoolFullError exception will be raised.
+        # If +timeout+ is not given, remove and return the head the
+        # queue if the number of available elements is strictly
+        # greater than the number of threads currently waiting (that
+        # is, don't jump ahead in line).  Otherwise, return nil.
         #
-        # Returns: an AbstractAdapter object.
+        # If +timeout+ is given, block if it there is no element
+        # available, waiting up to +timeout+ seconds for an element to
+        # become available.
         #
         # Raises:
-        # - ConnectionTimeoutError: no connection can be obtained from the pool.
-        def checkout
-          synchronize do
-            conn = acquire_connection
-            conn.lease
-            checkout_and_verify(conn)
-          end
-        end
-  
-        # Check-in a database connection back into the pool, indicating that you
-        # no longer need this connection.
-        #
-        # +conn+: an AbstractAdapter object, which was obtained by earlier by
-        # calling +checkout+ on this pool.
-        def checkin(conn)
+        # - ConnectionTimeoutError if +timeout+ is given and no element
+        # becomes available after +timeout+ seconds,
+        def poll(timeout = nil)
           synchronize do
-            conn.run_callbacks :checkin do
-              conn.expire
+            if timeout
+              no_wait_poll || wait_poll(timeout)
+            else
+              no_wait_poll
             end
-  
-            release conn
-  
-            @available.add conn
           end
         end
-  
-        # Remove a connection from the connection pool.  The connection will
-        # remain open and active but will no longer be managed by this pool.
-        def remove(conn)
+        
+        def num_available
           synchronize do
-            @connections.delete conn
-            @available.delete conn
-  
-            # FIXME: we might want to store the key on the connection so that removing
-            # from the reserved hash will be a little easier.
-            release conn
-  
-            @available.add checkout_new_connection if @available.any_waiting?
+            @queue.size
           end
         end
-  
-      
-  
+
         private
-  
-        # Acquire a connection by one of 1) immediately removing one
-        # from the queue of available connections, 2) creating a new
-        # connection if the pool is not at capacity, 3) waiting on the
-        # queue for a connection to become available.
-        #
-        # Raises:
-        # - ConnectionTimeoutError if a connection could not be acquired (FIXME:
-        #   why not ConnectionTimeoutError?
-        def acquire_connection
-          if conn = @available.poll
-            conn
-          elsif @connections.size < @size
-            checkout_new_connection
-          else
-            t0 = Time.now
-            begin
-              @available.poll(@checkout_timeout)
-            rescue ConnectionTimeoutError
-              msg = 'could not obtain a database connection within %0.3f seconds (waited %0.3f seconds)' %
-                [@checkout_timeout, Time.now - t0]
-              raise ConnectionTimeoutError, msg
+
+        def synchronize(&block)
+          @lock.synchronize(&block)
+        end
+
+        # Test if the queue currently contains any elements.
+        def any?
+          !@queue.empty?
+        end
+
+        # A thread can remove an element from the queue without
+        # waiting if an only if the number of currently available
+        # connections is strictly greater than the number of waiting
+        # threads.
+        def can_remove_no_wait?
+          @queue.size > @num_waiting
+        end
+
+        # Removes and returns the head of the queue if possible, or nil.
+        def remove
+          @queue.shift
+        end
+
+        # Remove and return the head the queue if the number of
+        # available elements is strictly greater than the number of
+        # threads currently waiting.  Otherwise, return nil.
+        def no_wait_poll
+          remove if can_remove_no_wait?
+        end
+
+        # Waits on the queue up to +timeout+ seconds, then removes and
+        # returns the head of the queue.
+        def wait_poll(timeout)
+          @num_waiting += 1
+
+          t0 = Time.now
+          elapsed = 0
+          loop do
+            @cond.wait(timeout - elapsed)
+
+            return remove if any?
+
+            elapsed = Time.now - t0
+            raise ActiveRecord::ConnectionTimeoutError if elapsed >= timeout
+          end
+        ensure
+          @num_waiting -= 1
+        end
+      end
+
+      # Every +frequency+ seconds, the reaper will call +reap+ on +pool+.
+      # A reaper instantiated with a nil frequency will never reap the
+      # connection pool.
+      #
+      # Configure the frequency by setting "reaping_frequency" in your
+      # database yaml file.
+      class Reaper
+        attr_reader :pool, :frequency
+
+        def initialize(pool, frequency)
+          @pool      = pool
+          @frequency = frequency
+        end
+
+        def run
+          return unless frequency
+          Thread.new(frequency, pool) { |t, p|
+            while true
+              sleep t
+              p.reap
             end
+          }
+        end
+      end
+
+      include MonitorMixin
+
+      attr_accessor :automatic_reconnect, :checkout_timeout, :dead_connection_timeout
+      attr_reader :spec, :connections, :size, :reaper
+
+      # Creates a new ConnectionPool object. +spec+ is a ConnectionSpecification
+      # object which describes database connection information (e.g. adapter,
+      # host name, username, password, etc), as well as the maximum size for
+      # this ConnectionPool.
+      #
+      # The default ConnectionPool maximum size is 5.
+      def initialize(spec)
+        super()
+
+        @spec = spec
+
+        # The cache of reserved connections mapped to threads
+        @reserved_connections = {}
+
+        @checkout_timeout = spec.config[:checkout_timeout] || 5
+        @dead_connection_timeout = spec.config[:dead_connection_timeout]
+        @reaper  = Reaper.new self, spec.config[:reaping_frequency]
+        @reaper.run
+
+        # default max pool size to 5
+        @size = (spec.config[:pool] && spec.config[:pool].to_i) || 5
+
+        @connections         = []
+        @automatic_reconnect = true
+
+        @available = Queue.new self
+      end
+
+      # Hack for tests to be able to add connections.  Do not call outside of tests
+      def insert_connection_for_test!(c) #:nodoc:
+        synchronize do
+          @connections << c
+          @available.add c
+        end
+      end
+
+      # Retrieve the connection associated with the current thread, or call
+      # #checkout to obtain one if necessary.
+      #
+      # #connection can be called any number of times; the connection is
+      # held in a hash keyed by the thread id.
+      def connection
+        synchronize do
+          @reserved_connections[current_connection_id] ||= checkout
+        end
+      end
+
+      # Is there an open connection that is being used for the current thread?
+      def active_connection?
+        synchronize do
+          @reserved_connections.fetch(current_connection_id) {
+            return false
+          }.in_use?
+        end
+      end
+
+      # Signal that the thread is finished with the current connection.
+      # #release_connection releases the connection-thread association
+      # and returns the connection to the pool.
+      def release_connection(with_id = current_connection_id)
+        synchronize do
+          conn = @reserved_connections.delete(with_id)
+          checkin conn if conn
+        end
+      end
+
+      # If a connection already exists yield it to the block. If no connection
+      # exists checkout a connection, yield it to the block, and checkin the
+      # connection when finished.
+      def with_connection
+        connection_id = current_connection_id
+        fresh_connection = true unless active_connection?
+        yield connection
+      ensure
+        release_connection(connection_id) if fresh_connection
+      end
+
+      # Returns true if a connection has already been opened.
+      def connected?
+        synchronize { @connections.any? }
+      end
+
+      # Disconnects all connections in the pool, and clears the pool.
+      def disconnect!
+        synchronize do
+          @reserved_connections = {}
+          @connections.each do |conn|
+            checkin conn
+            conn.disconnect!
           end
+          @connections = []
+          @available.clear
         end
-  
-        def release(conn)
-          thread_id = if @reserved_connections[current_connection_id] == conn
-            current_connection_id
-          else
-            @reserved_connections.keys.find { |k|
-              @reserved_connections[k] == conn
-            }
+      end
+
+      # Clears the cache which maps classes.
+      def clear_reloadable_connections!
+        synchronize do
+          @reserved_connections = {}
+          @connections.each do |conn|
+            checkin conn
+            conn.disconnect! if conn.requires_reloading?
+          end
+          @connections.delete_if do |conn|
+            conn.requires_reloading?
+          end
+          @available.clear
+          @connections.each do |conn|
+            @available.add conn
           end
-  
-          @reserved_connections.delete thread_id if thread_id
         end
-  
-        def new_connection
-          ActiveRecord::Base.send(spec.adapter_method, spec.config)
+      end
+
+      # clear_stale_cached imp from Rails 3.2, still using Threads. 
+      # Yes, we've created a monster. 
+      # Return any checked-out connections back to the pool by threads that
+      # are no longer alive.
+      def clear_stale_cached_connections!
+        keys = @reserved_connections.keys - Thread.list.find_all { |t|
+          t.alive?
+        }.map { |thread| thread.object_id }
+        keys.each do |key|
+          conn = @reserved_connections[key]
+          ActiveSupport::Deprecation.warn(<<-eowarn) if conn.in_use?
+Database connections will not be closed automatically, please close your
+database connection at the end of the thread by calling `close` on your
+connection.  For example: ActiveRecord::Base.connection.close
+          eowarn
+          checkin conn
+          @reserved_connections.delete(key)
         end
-  
-        def current_connection_id #:nodoc:
-          ActiveRecord::Base.connection_id ||= Thread.current.object_id
+      end
+
+      # Check-out a database connection from the pool, indicating that you want
+      # to use it. You should call #checkin when you no longer need this.
+      #
+      # This is done by either returning and leasing existing connection, or by
+      # creating a new connection and leasing it.
+      #
+      # If all connections are leased and the pool is at capacity (meaning the
+      # number of currently leased connections is greater than or equal to the
+      # size limit set), an ActiveRecord::PoolFullError exception will be raised.
+      #
+      # Returns: an AbstractAdapter object.
+      #
+      # Raises:
+      # - PoolFullError: no connection can be obtained from the pool.
+      def checkout
+        synchronize do
+          conn = acquire_connection
+          conn.lease
+          checkout_and_verify(conn)
         end
-  
-        def checkout_new_connection
-          raise ConnectionNotEstablished unless @automatic_reconnect
-  
-          c = new_connection
-          c.pool = self
-          @connections << c
-          c
+      end
+
+      # Check-in a database connection back into the pool, indicating that you
+      # no longer need this connection.
+      #
+      # +conn+: an AbstractAdapter object, which was obtained by earlier by
+      # calling +checkout+ on this pool.
+      def checkin(conn)
+        synchronize do
+          conn.run_callbacks :checkin do
+            conn.expire
+          end
+
+          release conn
+
+          @available.add conn
         end
-  
-        def checkout_and_verify(c)
-          c.run_callbacks :checkout do
-            c.verify!
+      end
+
+      # Remove a connection from the connection pool.  The connection will
+      # remain open and active but will no longer be managed by this pool.
+      def remove(conn)
+        synchronize do
+          @connections.delete conn
+          @available.delete conn
+
+          # FIXME: we might want to store the key on the connection so that removing
+          # from the reserved hash will be a little easier.
+          release conn
+
+          @available.add checkout_new_connection if @available.any_waiting?
+        end
+      end
+
+      # Removes dead connections from the pool.  A dead connection can occur
+      # if a programmer forgets to close a connection at the end of a thread
+      # or a thread dies unexpectedly.
+      def reap
+        synchronize do
+          stale = Time.now - @dead_connection_timeout
+          connections.dup.each do |conn|
+            remove conn if conn.in_use? && stale > conn.last_use && !conn.active?
           end
-          c
         end
       end
-  
-     
+
+      private
+
+      # Acquire a connection by one of 1) immediately removing one
+      # from the queue of available connections, 2) creating a new
+      # connection if the pool is not at capacity, 3) waiting on the
+      # queue for a connection to become available.
+      #
+      # Raises:
+      # - PoolFullError if a connection could not be acquired (FIXME:
+      #   why not ConnectionTimeoutError?
+      def acquire_connection
+        if conn = @available.poll
+          conn
+        elsif @connections.size < @size
+          checkout_new_connection
+        else
+          clear_stale_cached_connections!
+          
+          t0 = Time.now
+          
+          Rails.logger.info("POLLED_CHECKOUT: num avail connections: #{@available.num_available}; num waiting: #{@available.num_waiting}; total connections: #{@connections.size}")
+          begin
+            @available.poll(@checkout_timeout)
+          rescue ActiveRecord::ConnectionTimeoutError
+            msg = 'could not obtain a database connection within %0.3f seconds (waited %0.3f seconds)' %
+              [@checkout_timeout, Time.now - t0]
+            raise ActiveRecord::ConnectionTimeoutError, msg
+          end
+        end
+      end
+
+      def release(conn)
+        thread_id = if @reserved_connections[current_connection_id] == conn
+          current_connection_id
+        else
+          @reserved_connections.keys.find { |k|
+            @reserved_connections[k] == conn
+          }
+        end
+
+        @reserved_connections.delete thread_id if thread_id
+      end
+
+      def new_connection
+        ActiveRecord::Base.send(spec.adapter_method, spec.config)
+      end
+
+      def current_connection_id #:nodoc:
+        ActiveRecord::Base.connection_id ||= Thread.current.object_id
+      end
+
+      def checkout_new_connection
+        raise ConnectionNotEstablished unless @automatic_reconnect
+
+        c = new_connection
+        c.pool = self
+        @connections << c
+        c
+      end
+
+      def checkout_and_verify(c)
+        c.run_callbacks :checkout do
+          c.verify!
+        end
+        c
+      end
     end
-  end
   
 end  
   

data/app/controllers/search_methods/sfx4.rb

@@ -156,8 +156,8 @@ module SearchMethods
         
         extra_info_xml = Nokogiri::XML( sfx_obj["EXTRA_INFO_XML"] )
         
-        # Put SFX object id in rft.object_id, that's what SFX does. 
-        ctx.referent.set_metadata('object_id', sfx_obj["OBJECT_ID"])
+        # Put SFX object id in rft.object_id, that's what SFX does.
+        ctx.referent.set_metadata('object_id', sfx_obj["OBJECT_ID"].to_s )
         ctx.referent.set_metadata("jtitle", sfx_obj["TITLE_DISPLAY"] || "Unknown Title")
         
         issn = extra_info_xml.search("item[key=issn]").text

data/app/models/collection.rb

@@ -60,7 +60,7 @@ class Collection
   # Then actually executes the services that are dispatchable (queued).    
   def dispatch_services!
     queued_service_ids = prepare_for_dispatch!
-    
+        
     dispatch_foreground!(queued_service_ids)
         
     dispatch_background!(queued_service_ids)        
@@ -152,68 +152,70 @@ class Collection
   def prepare_for_dispatch!
     # Go through currently dispatched services, looking for timed out
     # services -- services still in progress that have taken too long,
-    # as well as service responses that are too old to be used.      
-    umlaut_request.dispatched_services.each do | ds |
-        # go through dispatched_services and set stil in progress but too long to failed temporary
-        if ( (ds.status == DispatchedService::InProgress ||
-              ds.status == DispatchedService::Queued ) &&
-              (Time.now - ds.updated_at) > self.background_service_timeout)
-
-              ds.store_exception( Exception.new("background service timed out (took longer than #{self.background_service_timeout} to run); thread assumed dead.")) unless ds.exception_info
-              # Fail it temporary, it'll be run again. 
-              ds.status = DispatchedService::FailedTemporary
-              ds.save!
-              logger.warn("Background service timed out, thread assumed dead. #{umlaut_request.id} / #{ds.service_id}")
-         end
-         
-        # go through dispatched_services and delete:
-        # 1) old completed dispatches, too old to use. 
-        # 2) failedtemporary dispatches that are older than our resurrection time
-        # -> And all responses associated with those dispatches. 
-        # After being deleted, they'll end up re-queued. 
-        if ( (ds.completed? && completed_dispatch_expired?(ds) ) ||
-             (  ds.status == DispatchedService::FailedTemporary &&
-               (Time.now - ds.updated_at) > self.requeue_failedtemporary_services 
+    # as well as service responses that are too old to be used.
+    queued_service_ids = []
+    DispatchedService.transaction do
+      umlaut_request.dispatched_services.each do | ds |
+          # go through dispatched_services and set stil in progress but too long to failed temporary
+          if ( (ds.status == DispatchedService::InProgress ||
+                ds.status == DispatchedService::Queued ) &&
+                (Time.now - ds.updated_at) > self.background_service_timeout)
+  
+                ds.store_exception( Exception.new("background service timed out (took longer than #{self.background_service_timeout} to run); thread assumed dead.")) unless ds.exception_info
+                # Fail it temporary, it'll be run again. 
+                ds.status = DispatchedService::FailedTemporary
+                ds.save!
+                logger.warn("Background service timed out, thread assumed dead. #{umlaut_request.id} / #{ds.service_id}")
+           end
+           
+          # go through dispatched_services and delete:
+          # 1) old completed dispatches, too old to use. 
+          # 2) failedtemporary dispatches that are older than our resurrection time
+          # -> And all responses associated with those dispatches. 
+          # After being deleted, they'll end up re-queued. 
+          if ( (ds.completed? && completed_dispatch_expired?(ds) ) ||
+               (  ds.status == DispatchedService::FailedTemporary &&
+                 (Time.now - ds.updated_at) > self.requeue_failedtemporary_services 
+                )
               )
-            )
+              
+            # Need to expire. Delete all the service responses, and
+            # the DispatchedService record, and service will be automatically
+            # run again.
+            serv_id = ds.service_id
             
-          # Need to expire. Delete all the service responses, and
-          # the DispatchedService record, and service will be automatically
-          # run again.
-          serv_id = ds.service_id
-          
-          umlaut_request.service_responses.each do |response|            
-            if response.service_id == serv_id
-              umlaut_request.service_responses.delete(response)              
-              response.destroy
+            umlaut_request.service_responses.each do |response|            
+              if response.service_id == serv_id
+                umlaut_request.service_responses.delete(response)              
+                response.destroy
+              end
             end
+            
+            umlaut_request.dispatched_services.delete(ds)
+            ds.destroy
           end
+      end
           
-          umlaut_request.dispatched_services.delete(ds)
-          ds.destroy
-        end
-    end
-        
-    # Queue any services without a dispatch marker at all, keeping
-    # track of queued services, already existing or newly created. 
-    
-    # Just in case, we're going to refetch dispatched_services from the db,
-    # in case some other http request or background service updated things
-    # recently. 
-    umlaut_request.dispatched_services.reset
-    
-    queued_service_ids = []
-    self.get_service_definitions.each do |service|
-      service_id = service['service_id']
-      # use in-memory #to_a search, don't go to db each time!
-      if found = umlaut_request.dispatched_services.to_a.find {|s| s.service_id == service_id}
-        queued_service_ids.push(service_id) if found.status == DispatchedService::Queued
-      else
-        umlaut_request.new_dispatch_object!(service_id, DispatchedService::Queued).save!
-        queued_service_ids.push(service_id)
-      end        
+      # Queue any services without a dispatch marker at all, keeping
+      # track of queued services, already existing or newly created. 
+      
+      # Just in case, we're going to refetch dispatched_services from the db,
+      # in case some other http request or background service updated things
+      # recently. 
+      umlaut_request.dispatched_services.reset
+      
+      self.get_service_definitions.each do |service|
+        service_id = service['service_id']
+        # use in-memory #to_a search, don't go to db each time!
+        if found = umlaut_request.dispatched_services.to_a.find {|s| s.service_id == service_id}
+          queued_service_ids.push(service_id) if found.status == DispatchedService::Queued
+        else
+          umlaut_request.new_dispatch_object!(service_id, DispatchedService::Queued).save!
+          queued_service_ids.push(service_id)
+        end        
+      end
     end
-    
+      
     return queued_service_ids
   end
   

data/app/models/service_response.rb

@@ -109,6 +109,17 @@ class ServiceResponse < ActiveRecord::Base
     self.service.view_data_from_service_type(self)
   end
   
+  
+  def service_data
+    # Fix weird-ass char encoding bug with AR serialize and hashes.
+    # https://github.com/rails/rails/issues/6538
+    data = super
+    if data.kind_of? Hash
+      data.values.each {|v| v.force_encoding "UTF-8"  if v.respond_to? :force_encoding  }
+    end
+    return data
+  end
+  
     # Should take a ServiceTypeValue object, or symbol name of
   # ServiceTypeValue object. 
   def service_type_value=(value)

data/app/models/service_wave.rb

@@ -95,7 +95,7 @@ class ServiceWave
             
             # Log it too, although experience shows it may never make it to the 
             # log for mysterious reasons. 
-            Rails.logger.error(TermColor.color("Umlaut: Threaded service raised exception.", :red, true) + "Service: #{service.service_id}, #{e}\n  #{clean_backtrace(e).join("\n  ")}")
+            Rails.logger.error(TermColor.color("Umlaut: Threaded service raised exception.", :red, true) + " Service: #{service.service_id}, #{e.class} #{e.message}\n  #{clean_backtrace(e).join("\n  ")}")
             
             # And stick it in a thread variable too
             Thread.current[:exception] = e                      

data/db/migrate/02_umlaut_add_service_response_index.rb

@@ -0,0 +1,9 @@
+class UmlautAddServiceResponseIndex < ActiveRecord::Migration
+  def up
+    add_index "service_responses", ["request_id"]
+  end
+
+  def down
+    remove_index "service_responses", ["request_id"]
+  end
+end

data/lib/service_adaptors/hathi_trust.rb

@@ -131,7 +131,7 @@ class HathiTrust < Service
     isbn = isbn.gsub(/[\-\[\]]/, '') unless isbn.blank?
     
     oclcnum = get_identifier(:info, "oclcnum", rft)
-    oclcnum = oclnum.gsub(/[\-\[\]]/, '') unless oclcnum.blank?
+    oclcnum = oclcnum.gsub(/[\-\[\]]/, '') unless oclcnum.blank?
     
     lccn = get_lccn(rft)
     lccn = lccn.gsub(/[\-\[\]]/, '') unless lccn.blank?

data/lib/tasks/umlaut_migrate_permalinks.rake

@@ -39,7 +39,7 @@ namespace :umlaut do
       # dynamically create a model for Permalinks in old
       # db, set up with connection info to old db. 
       OldPermalink = Class.new(ActiveRecord::Base) do
-        self.set_table_name("permalinks")
+        self.table_name = "permalinks"
         # just to be safe
         def read_only?
           true

data/lib/umlaut.rb

@@ -38,9 +38,9 @@ module Umlaut
     
     # Patch with fixed 'fair' version of ConnectionPool, see 
     # active_record_patch/connection_pool.rb
-    initializer("#{engine_name}.patch_connection_pool", :before => "active_record.initialize_database") do |app|
+    #initializer("#{engine_name}.patch_connection_pool", :before => "active_record.initialize_database") do |app|
       load File.join(self.root, "active_record_patch", "connection_pool.rb")
-    end
+    #end
     
          
     

data/lib/umlaut/version.rb

@@ -1,3 +1,3 @@
 module Umlaut
-  VERSION = "3.0.0beta9"
+  VERSION = "3.0.0beta10"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: umlaut
 version: !ruby/object:Gem::Version
-  version: 3.0.0beta9
+  version: 3.0.0beta10
   prerelease: 5
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-29 00:00:00.000000000 Z
+date: 2012-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -318,7 +318,6 @@ files:
 - app/models/referent_value.rb
 - app/models/service_type_value.rb
 - app/models/service_store.rb
-- app/models/#Untitled-1#
 - app/models/sfx_db.rb
 - app/models/collection.rb
 - app/models/dispatched_service.rb
@@ -377,6 +376,7 @@ files:
 - app/helpers/resolve_helper.rb
 - app/helpers/search_helper.rb
 - db/seeds.rb
+- db/migrate/02_umlaut_add_service_response_index.rb
 - db/migrate/01_umlaut_init.rb
 - db/orig_fixed_data/service_type_values.yml
 - lib/opensearch_query.rb
@@ -558,7 +558,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1502362427621060492
+      hash: 2903597674342542131
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: