rubyrep 1.0.5 → 1.0.6

data/lib/rubyrep/replication_run.rb

@@ -8,6 +8,9 @@ module RR
     # The current Session object
     attr_accessor :session
 
+    # The current TaskSweeper
+    attr_accessor :sweeper
+
     # Returns the current ReplicationHelper; creates it if necessary
     def helper
       @helper ||= ReplicationHelper.new(self)
@@ -22,22 +25,29 @@ module RR
     # Executes the replication run.
     def run
       return unless [:left, :right].any? do |database|
-        Timeout::timeout(session.configuration.options[:database_connection_timeout]) do
-          session.send(database).select_one(
+        changes_pending = false
+        t = Thread.new do
+          changes_pending = session.send(database).select_one(
             "select id from #{session.configuration.options[:rep_prefix]}_pending_changes"
           ) != nil
         end
+        t.join session.configuration.options[:database_connection_timeout]
+        changes_pending
       end
+
+      loaders = LoggedChangeLoaders.new(session)
+
       begin
         success = false
         replicator # ensure that replicator is created and has chance to validate settings
 
         loop do
           begin
-            session.reload_changes # ensure the cache of change log records is up-to-date
-            diff = ReplicationDifference.new session
+            loaders.update # ensure the cache of change log records is up-to-date
+            diff = ReplicationDifference.new loaders
             diff.load
             break unless diff.loaded?
+            raise "Replication run timed out" if sweeper.terminated?
             replicator.replicate_difference diff if diff.type != :no_diff
           rescue Exception => e
             helper.log_replication_outcome diff, e.message,
@@ -50,10 +60,23 @@ module RR
       end
     end
 
+    # Installs the current sweeper into the database connections
+    def install_sweeper
+      [:left, :right].each do |database|
+        unless session.send(database).respond_to?(:sweeper)
+          session.send(database).send(:extend, NoisyConnection)
+        end
+        session.send(database).sweeper = sweeper
+      end
+    end
+
     # Creates a new ReplicationRun instance.
     # * +session+: the current Session
-    def initialize(session)
+    # * +sweeper+: the current TaskSweeper
+    def initialize(session, sweeper)
       self.session = session
+      self.sweeper = sweeper
+      install_sweeper
     end
   end
-end
+end

data/lib/rubyrep/replication_runner.rb

@@ -108,8 +108,12 @@ EOS
       until termination_requested do
         begin
           session.refresh
-          run = ReplicationRun.new session
-          run.run
+          timeout = session.configuration.options[:database_connection_timeout]
+          terminated = TaskSweeper.timeout(timeout) do |sweeper|
+            run = ReplicationRun.new session, sweeper
+            run.run
+          end.terminated?
+          raise "replication run timed out" if terminated
         rescue Exception => e
           now = Time.now.iso8601
           $stderr.puts "#{now} Exception caught: #{e}"

data/lib/rubyrep/session.rb

@@ -83,32 +83,7 @@ module RR
       @table_map[db_arm][table] || table
     end
     
-    # Does the actual work of establishing a database connection
-    # db_arm:: should be either :left or :right
-    # config:: the rubyrep Configuration
-    def db_connect(db_arm, config)
-      arm_config = config.send db_arm
-      @proxies[db_arm] = DatabaseProxy.new
-      @connections[db_arm] = @proxies[db_arm].create_session arm_config
-    end
-    
-    # Does the actual work of establishing a proxy connection
-    # db_arm:: should be either :left or :right
-    # config:: the rubyrep Configuration
-    def proxy_connect(db_arm, config)
-      arm_config = config.send db_arm
-      if arm_config.include? :proxy_host 
-        drb_url = "druby://#{arm_config[:proxy_host]}:#{arm_config[:proxy_port]}"
-        @proxies[db_arm] = DRbObject.new nil, drb_url
-      else
-        # If one connection goes through a proxy, so has the other one.
-        # So if necessary, create a "fake" proxy
-        @proxies[db_arm] = DatabaseProxy.new
-      end
-      @connections[db_arm] = @proxies[db_arm].create_session arm_config
-    end
-    
-    # True if proxy connections are used
+    # Returns +true+ if proxy connections are used
     def proxied?
       [configuration.left, configuration.right].any? \
         {|arm_config| arm_config.include? :proxy_host}
@@ -144,9 +119,90 @@ module RR
       end
     end
 
-    # Refreshes (reestablish if no more active) the database connections.
+    # Returns +true+ if the specified database connection is not alive.
+    # * +database+: target database (either +:left+ or :+right+)
+    def database_unreachable?(database)
+      unreachable = true
+      Thread.new do
+        begin
+          if send(database) && send(database).select_one("select 1+1 as x")['x'].to_i == 2
+            unreachable = false # database is actually reachable
+          end
+        end rescue nil
+      end.join configuration.options[:database_connection_timeout]
+      unreachable
+    end
+
+    # Disconnnects the specified database
+    # * +database+: the target database (either :+left+ or :+right+)
+    def disconnect_database(database)
+      proxy, connection = @proxies[database], @connection[database]
+      @proxies[database] = nil
+      @connections[database] = nil
+      if proxy
+        proxy.destroy_session(connection)
+      end
+    end
+
+    # Refreshes both database connections
     def refresh
-      [:left, :right].each {|database| send(database).refresh}
+      [:left, :right].each {|database| refresh_database_connection database}
+    end
+
+    # Refreshes the specified database connection.
+    # (I. e. reestablish if not active anymore.)
+    # * +database+: target database (either :+left+ or :+right+)
+    def refresh_database_connection(database)
+      if database_unreachable?(database)
+        # step 1: disconnect both database connection (if still possible)
+        begin
+          Thread.new do
+            disconnect_database database rescue nil
+          end.join configuration.options[:database_connection_timeout]
+        end
+
+        connect_exception = nil
+        # step 2: try to reconnect the database
+        Thread.new do
+          begin
+            connect_database database
+          rescue Exception => e
+            # save exception so it can be rethrown outside of the thread
+            connect_exception = e
+          end
+        end.join configuration.options[:database_connection_timeout]
+        raise connect_exception if connect_exception
+
+        # step 3: verify if database connections actually work (to detect silent connection failures)
+        if database_unreachable?(database)
+          raise "no connection to '#{database}' database"
+        end
+      end
+    end
+
+    # Set up the (proxied or direct) database connections to the specified
+    # database.
+    # * +database+: the target database (either :+left+ or :+right+)
+    def connect_database(database)
+      if configuration.left == configuration.right and database == :right
+        # If both database configurations point to the same database
+        # then don't create the database connection twice.
+        # Assumes that the left database is always connected before the right one.
+        self.right = self.left
+      else
+        # Connect the database / proxy
+        arm_config = configuration.send database
+        if arm_config.include? :proxy_host
+          drb_url = "druby://#{arm_config[:proxy_host]}:#{arm_config[:proxy_port]}"
+          @proxies[database] = DRbObject.new nil, drb_url
+        else
+          # Create fake proxy
+          @proxies[database] = DatabaseProxy.new
+        end
+        @connections[database] = @proxies[database].create_session arm_config
+
+        send(database).manual_primary_keys = manual_primary_keys(database)
+      end
     end
         
     # Creates a new rubyrep session with the provided Configuration
@@ -157,21 +213,7 @@ module RR
       # Keep the database configuration for future reference
       self.configuration = config
 
-      # Determine method of connection (either 'proxy_connect' or 'db_connect'
-      connection_method = proxied? ? :proxy_connect : :db_connect
-      
-      # Connect the left database / proxy
-      self.send connection_method, :left, configuration
-      left.manual_primary_keys = manual_primary_keys(:left)
-      
-      # If both database configurations point to the same database
-      # then don't create the database connection twice
-      if configuration.left == configuration.right
-        self.right = self.left
-      else
-        self.send connection_method, :right, configuration
-        right.manual_primary_keys = manual_primary_keys(:right)
-      end  
+      refresh
     end
   end
 end

data/lib/rubyrep/task_sweeper.rb

@@ -0,0 +1,77 @@
+module RR
+
+  # Monitors and cancels stalled tasks
+  class TaskSweeper
+
+    # Executes the give block in a separate Thread.
+    # Returns if block is finished or stalled.
+    # The block must call regular #ping to announce it is not stalled.
+    # * +timeout_period+:
+    #   Maximum time (in seonds) without ping, after which a task is considered stalled.
+    # Returns the created sweeper (allows checking if task was terminated).
+    def self.timeout(timeout_period)
+      sweeper = TaskSweeper.new(timeout_period)
+      sweeper.send(:timeout) {yield sweeper}
+      sweeper
+    end
+
+    # Time in seconds after which a task is considered stalled. Timer is reset
+    # by calling #ping.
+    attr_accessor :timeout_period
+
+    # Must be called by the executed task to announce it is still alive
+    def ping
+      self.last_ping = Time.now
+    end
+
+    # Returns +true+ if the task was timed out.
+    # The terminated task is expected to free all resources and exit.
+    def terminated?
+      terminated
+    end
+
+    # Waits without timeout till the task executing thread is finished
+    def join
+      thread && thread.join
+    end
+
+    # Creates a new TaskSweeper
+    # * +timeout_period+: timeout value in seconds
+    def initialize(timeout_period)
+      self.timeout_period = timeout_period
+      self.terminated = false
+      self.last_ping = Time.now
+    end
+
+    protected
+
+    # Time of last ping
+    attr_accessor :last_ping
+
+    # Set to +true+ if the executed task has timed out
+    attr_accessor :terminated
+
+    # The task executing thread
+    attr_accessor :thread
+
+    # Executes the given block and times it out if stalled.
+    def timeout
+      exception = nil
+      self.thread = Thread.new do
+        begin
+          yield
+        rescue Exception => e
+          # save exception so it can be rethrown outside of the thread
+          exception = e
+        end
+      end
+      while self.thread.join(self.timeout_period) == nil do
+        if self.last_ping < Time.now - self.timeout_period
+          self.terminated = true
+          break
+        end
+      end
+      raise exception if exception
+    end
+  end
+end

data/lib/rubyrep/version.rb

@@ -2,7 +2,7 @@ module RR #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 1
     MINOR = 0
-    TINY  = 5
+    TINY  = 6
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/rubyrep.rb

@@ -42,15 +42,18 @@ require 'syncers/syncers'
 require 'syncers/two_way_syncer'
 require 'sync_runner'
 require 'trigger_mode_switcher'
+require 'logged_change_loader'
 require 'logged_change'
 require 'replication_difference'
 require 'replication_helper'
 require 'replicators/replicators'
 require 'replicators/two_way_replicator'
+require 'task_sweeper'
 require 'replication_run'
 require 'replication_runner'
 require 'uninstall_runner'
 require 'generate_runner'
+require 'noisy_connection'
 
 Dir["#{File.dirname(__FILE__)}/rubyrep/connection_extenders/*.rb"].each do |extender|
   # jdbc_extender.rb is only loaded if we are running on jruby

data/spec/logged_change_loader_spec.rb

@@ -0,0 +1,68 @@
+require File.dirname(__FILE__) + '/spec_helper.rb'
+
+include RR
+
+describe LoggedChangeLoaders do
+  before(:each) do
+    Initializer.configuration = standard_config
+  end
+
+  it "initializers should create both logged change loaders" do
+    session = Session.new
+    loaders = LoggedChangeLoaders.new(session)
+    loaders[:left].session.should == session
+    loaders[:left].database.should == :left
+    loaders[:right].database.should == :right
+  end
+
+  it "update should execute a forced update of both logged change loaders" do
+    session = Session.new
+    loaders = LoggedChangeLoaders.new(session)
+    loaders[:left].should_receive(:update).with(:forced => true)
+    loaders[:right].should_receive(:update).with(:forced => true)
+    loaders.update
+  end
+
+end
+
+describe LoggedChangeLoader do
+  before(:each) do
+    Initializer.configuration = standard_config
+  end
+
+  # Note:
+  # LoggedChangeLoader is a helper for LoggedChange.
+  # It is tested through the specs for LoggedChange.
+
+  it "oldest_change_time should return nil if there are no changes" do
+    session = Session.new
+    session.left.execute "delete from rr_pending_changes"
+    loader = LoggedChangeLoader.new session, :left
+    loader.oldest_change_time.should be_nil
+  end
+
+  it "oldest_change_time should return the time of the oldest change" do
+    session = Session.new
+    session.left.begin_db_transaction
+    begin
+      time = Time.now
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'left_table',
+        'change_key' => 'id|1',
+        'change_type' => 'I',
+        'change_time' => time
+      }
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'left_table',
+        'change_key' => 'id|2',
+        'change_type' => 'I',
+        'change_time' => 100.seconds.from_now
+      }
+      loader = LoggedChangeLoader.new session, :left
+      loader.oldest_change_time.should.to_s == time.to_s
+    ensure
+      session.left.rollback_db_transaction
+    end
+  end
+
+end

data/spec/logged_change_spec.rb

@@ -9,7 +9,8 @@ describe LoggedChange do
 
   it "initialize should store session and database" do
     session = Session.new
-    change = LoggedChange.new session, :left
+    loader = LoggedChangeLoader.new session, :left
+    change = LoggedChange.new loader
     change.session.should == session
     change.database.should == :left
   end
@@ -37,7 +38,8 @@ describe LoggedChange do
         'change_type' => 'I',
         'change_time' => Time.now
       }
-      change = LoggedChange.new session, :left
+      loader = LoggedChangeLoader.new session, :left
+      change = LoggedChange.new loader
       change.load_specified 'left_table', {'id' => '2'}
 
       change.table.should == 'left_table'
@@ -62,7 +64,8 @@ describe LoggedChange do
         'change_type' => 'I',
         'change_time' => Time.now
       }
-      change = LoggedChange.new session, :left
+      loader = LoggedChangeLoader.new session, :left
+      change = LoggedChange.new loader
       change.load_specified 'scanner_records', {'id1' => 1, 'id2' => 2}
 
       change.table.should == 'scanner_records'
@@ -83,7 +86,8 @@ describe LoggedChange do
         'change_type' => 'I',
         'change_time' => Time.now
       }
-      change = LoggedChange.new session, :left
+      loader = LoggedChangeLoader.new session, :left
+      change = LoggedChange.new loader
       change.load_specified 'left_table', {'id' => 1}
 
       session.left.
@@ -113,7 +117,8 @@ describe LoggedChange do
         'change_type' => 'U',
         'change_time' => t2
       }
-      change = LoggedChange.new session, :left
+      loader = LoggedChangeLoader.new session, :left
+      change = LoggedChange.new loader
       change.load_specified 'left_table', {'id' => 1}
 
       change.first_changed_at.to_s.should == t1.to_s
@@ -141,7 +146,8 @@ describe LoggedChange do
         'change_type' => 'U',
         'change_time' => Time.now
       }
-      change = LoggedChange.new session, :left
+      loader = LoggedChangeLoader.new session, :left
+      change = LoggedChange.new loader
       change.load_specified 'left_table', {'id' => 1}
 
       change.type.should == :update
@@ -175,7 +181,8 @@ describe LoggedChange do
         'change_type' => 'D',
         'change_time' => Time.now
       }
-      change = LoggedChange.new session, :left
+      loader = LoggedChangeLoader.new session, :left
+      change = LoggedChange.new loader
       change.load_specified 'left_table', {'id' => '1'}
 
       change.type.should == :no_change
@@ -215,7 +222,8 @@ describe LoggedChange do
         'change_type' => 'U',
         'change_time' => Time.now
       }
-      change = LoggedChange.new session, :left
+      loader = LoggedChangeLoader.new session, :left
+      change = LoggedChange.new loader
       change.load_specified 'left_table', {'id' => '1'}
       change.type.should == :insert
       change.key.should == {'id' => '2'}
@@ -233,8 +241,8 @@ describe LoggedChange do
         'change_type' => 'I',
         'change_time' => Time.now
       }
-      session.reload_changes
-      change = LoggedChange.new session, :left
+      loader.update :forced => true
+      change = LoggedChange.new loader
       change.load_specified 'left_table', {'id' => '5'}
       change.type.should == :update
       change.key.should == {'id' => '5'}
@@ -254,7 +262,8 @@ describe LoggedChange do
         'change_type' => 'I',
         'change_time' => Time.now
       }
-      change = LoggedChange.new session, :left
+      loader = LoggedChangeLoader.new session, :left
+      change = LoggedChange.new loader
       change.load_specified 'scanner_records', {'id' => '1'}
 
       change.table.should == 'scanner_records'
@@ -275,7 +284,8 @@ describe LoggedChange do
     session = Session.new
     session.left.begin_db_transaction
     begin
-      change = LoggedChange.new session, :left
+      loader = LoggedChangeLoader.new session, :left
+      change = LoggedChange.new loader
       change.load_specified 'scanner_records', {'id' => '1'}
 
       change.table.should == 'scanner_records'
@@ -307,7 +317,8 @@ describe LoggedChange do
         'change_type' => 'U',
         'change_time' => Time.now
       }
-      change = LoggedChange.new session, :left
+      loader = LoggedChangeLoader.new session, :left
+      change = LoggedChange.new loader
       change.load_specified 'left_table', {'id' => '1'}
       session.left.insert_record 'rr_pending_changes', {
         'change_table' => 'left_table',
@@ -315,7 +326,7 @@ describe LoggedChange do
         'change_type' => 'D',
         'change_time' => Time.now
       }
-      session.reload_changes
+      loader.update :forced => true
       change.load
 
       change.table.should == 'left_table'
@@ -341,7 +352,8 @@ describe LoggedChange do
         'change_type' => 'U',
         'change_time' => Time.now
       }
-      change = LoggedChange.new session, :left
+      loader = LoggedChangeLoader.new session, :left
+      change = LoggedChange.new loader
       change.load_specified 'left_table', {'id' => '1'}
       session.left.insert_record 'rr_pending_changes', {
         'change_table' => 'left_table',
@@ -350,7 +362,7 @@ describe LoggedChange do
         'change_type' => 'U',
         'change_time' => Time.now
       }
-      session.reload_changes
+      loader.update :forced => true
       change.load
 
       change.table.should == 'left_table'
@@ -362,39 +374,9 @@ describe LoggedChange do
     end
   end
 
-  it "oldest_change_time should return nil if there are no changes" do
-    session = Session.new
-    session.left.execute "delete from rr_pending_changes"
-    change = LoggedChange.new session, :left
-    change.oldest_change_time.should be_nil
-  end
-
-  it "oldest_change_time should return the time of the oldest change" do
-    session = Session.new
-    session.left.begin_db_transaction
-    begin
-      time = Time.now
-      session.left.insert_record 'rr_pending_changes', {
-        'change_table' => 'left_table',
-        'change_key' => 'id|1',
-        'change_type' => 'I',
-        'change_time' => time
-      }
-      session.left.insert_record 'rr_pending_changes', {
-        'change_table' => 'left_table',
-        'change_key' => 'id|2',
-        'change_type' => 'I',
-        'change_time' => 100.seconds.from_now
-      }
-      change = LoggedChange.new session, :left
-      change.oldest_change_time.should.to_s == time.to_s
-    ensure
-      session.left.rollback_db_transaction
-    end
-  end
-
   it "key_from_raw_key should return the correct column_name => value hash for the given key" do
-    change = LoggedChange.new Session.new, :left
+    loader = LoggedChangeLoader.new Session.new, :left
+    change = LoggedChange.new loader
     change.key_to_hash("a|1|b|2").should == {
       'a' => '1',
       'b' => '2'
@@ -402,7 +384,8 @@ describe LoggedChange do
   end
 
   it "key_from_raw_key should work with multi character key_sep strings" do
-    change = LoggedChange.new Session.new, :left
+    loader = LoggedChangeLoader.new Session.new, :left
+    change = LoggedChange.new loader
     change.stub!(:key_sep).and_return('BLA')
     change.key_to_hash("aBLA1BLAbBLA2").should == {
       'a' => '1',
@@ -411,7 +394,8 @@ describe LoggedChange do
   end
 
   it "load_oldest should not load a change if none available" do
-    change = LoggedChange.new Session.new, :left
+    loader = LoggedChangeLoader.new Session.new, :left
+    change = LoggedChange.new loader
     change.should_not_receive :load_specified
     change.load_oldest
   end
@@ -432,7 +416,8 @@ describe LoggedChange do
         'change_type' => 'I',
         'change_time' => Time.now
       }
-      change = LoggedChange.new session, :left
+      loader = LoggedChangeLoader.new session, :left
+      change = LoggedChange.new loader
       change.load_oldest
 
       change.key.should == {'id' => '1'}
@@ -463,7 +448,8 @@ describe LoggedChange do
         'change_type' => 'I',
         'change_time' => Time.now
       }
-      change = LoggedChange.new session, :left
+      loader = LoggedChangeLoader.new session, :left
+      change = LoggedChange.new loader
       change.load_oldest
 
       change.type.should == :insert
@@ -473,8 +459,12 @@ describe LoggedChange do
     end
   end
 
-  it "to_yaml should blank out session" do
-    change = LoggedChange.new :dummy_session, :left
-    change.to_yaml.should_not =~ /session/
+  it "to_yaml should blank out session and loader" do
+    session = Session.new
+    loader = LoggedChangeLoader.new session, :left
+    change = LoggedChange.new loader
+    yaml = change.to_yaml
+    yaml.should_not =~ /session/
+    yaml.should_not =~ /loader/
   end
 end