rubyrep 1.0.5 → 1.0.6

data/History.txt

@@ -1,3 +1,9 @@
+== 1.0.6 2009-07-25
+
+* Bug fix: do not assume anymore that replication log events appear in sequential order
+* Feature: increased resilience of rubyrep to network timeouts
+* Feature: introduced :after_infrastructure_setup handler
+
 == 1.0.5 2009-07-03
 
 * Bug fix: rubyrep replication runs should survive update or delete attempts rejected by the database.

data/Manifest.txt

@@ -28,6 +28,8 @@ lib/rubyrep/generate_runner.rb
 lib/rubyrep/initializer.rb
 lib/rubyrep/log_helper.rb
 lib/rubyrep/logged_change.rb
+lib/rubyrep/logged_change_loader.rb
+lib/rubyrep/noisy_connection.rb
 lib/rubyrep/proxied_table_scan.rb
 lib/rubyrep/proxy_block_cursor.rb
 lib/rubyrep/proxy_connection.rb
@@ -60,6 +62,7 @@ lib/rubyrep/table_scan_helper.rb
 lib/rubyrep/table_sorter.rb
 lib/rubyrep/table_spec_resolver.rb
 lib/rubyrep/table_sync.rb
+lib/rubyrep/task_sweeper.rb
 lib/rubyrep/trigger_mode_switcher.rb
 lib/rubyrep/type_casting_cursor.rb
 lib/rubyrep/uninstall_runner.rb
@@ -89,7 +92,9 @@ spec/dolphins.jpg
 spec/generate_runner_spec.rb
 spec/initializer_spec.rb
 spec/log_helper_spec.rb
+spec/logged_change_loader_spec.rb
 spec/logged_change_spec.rb
+spec/noisy_connection_spec.rb
 spec/postgresql_replication_spec.rb
 spec/postgresql_schema_support_spec.rb
 spec/postgresql_support_spec.rb
@@ -126,6 +131,7 @@ spec/table_scan_spec.rb
 spec/table_sorter_spec.rb
 spec/table_spec_resolver_spec.rb
 spec/table_sync_spec.rb
+spec/task_sweeper_spec.rb
 spec/trigger_mode_switcher_spec.rb
 spec/two_way_replicator_spec.rb
 spec/two_way_syncer_spec.rb

data/lib/rubyrep/configuration.rb

@@ -112,6 +112,20 @@ module RR
     # * :+replication_interval+: time in seconds between replication runs
     # * :+database_connection_timeout+:
     #   Time in seconds after which database connections time out.
+    # * :+:after_infrastructure_setup+:
+    #   A Proc that is called after the replication infrastructure tables are
+    #   set up. Useful to e. g. tweak the access settings for the table.
+    #   The block is called with the current Session object.
+    #   The block is called every time replication is started, even if the
+    #   the infrastructure tables already existed.
+    #
+    # Example of an :+after_infrastructure_setup+ handler:
+    #   lambda do |session|
+    #     [:left, :right].each do |database|
+    #       session.send(left).execute \
+    #         "GRANT SELECT, UPDATE, INSERT ON rr_pending_changes TO scott"
+    #     end
+    #   end
     attr_reader :options
     
     # Merges the specified +options+ hash into the existing options

data/lib/rubyrep/logged_change.rb

@@ -1,184 +1,5 @@
 module RR
   
-  class Session
-
-    # Returns the +LoggedChangeLoader+ of the specified database.
-    # * database: either :+left+ or :+right+
-    def change_loader(database)
-      @change_loaders ||= {}
-      unless change_loader = @change_loaders[database]
-        change_loader = @change_loaders[database] = LoggedChangeLoader.new(self, database)
-      end
-      change_loader
-    end
-
-    # Forces an update of the change log cache
-    def reload_changes
-      change_loader(:left).update :forced => true
-      change_loader(:right).update :forced => true
-    end
-
-  end
-
-  # Caches the entries in the change log table
-  class LoggedChangeLoader
-
-    # The current +Session+.
-    attr_accessor :session
-
-    # The current +ProxyConnection+.
-    attr_accessor :connection
-
-    # Index to the next unprocessed change in the +change_array+.
-    attr_accessor :current_index
-
-    # ID of the last cached change log record.
-    attr_accessor :current_id
-
-    # Array with all cached changes.
-    # Processed change log records are replaced with +nil+.
-    attr_accessor :change_array
-
-    # Tree (hash) structure for fast access to all cached changes.
-    # First level of tree:
-    # * key: table name
-    # * value: 2nd level tree
-    # 2nd level tree:
-    # * key: the change_key value of the according change log records.
-    # * value:
-    #   An array of according change log records (column_name => value hash).
-    #   Additional entry of each change log hash:
-    #   * key: 'array_index'
-    #   * value: index to the change log record in +change_array+
-    attr_accessor :change_tree
-
-    # Date of last update of the cache
-    attr_accessor :last_updated
-
-    # Initializes / resets the cache.
-    def init_cache
-      self.change_tree = {}
-      self.change_array = []
-      self.current_index = 0
-    end
-    private :init_cache
-
-    # Create a new change log record cache.
-    # * +session+: The current +Session+
-    # * +database+: Either :+left+ or :+right+
-    def initialize(session, database)
-      self.session = session
-      self.connection = session.send(database)
-
-      init_cache
-      self.current_id = -1
-      self.last_updated = 1.year.ago
-    end
-
-    # Updates the cache.
-    # Options is a hash determining when the update is actually executed:
-    # * :+expire_time+: cache is older than the given number of seconds
-    # * :+forced+: if +true+ update the cache even if not yet expired
-    def update(options = {:forced => false, :expire_time => 1})
-      return unless options[:forced] or Time.now - self.last_updated >= options[:expire_time]
-      
-      self.last_updated = Time.now
-
-      # First, let's use a LIMIT clause (via :row_buffer_size option) to verify
-      # if there are any pending changes.
-      # (If there are many pending changes, this is (at least with PostgreSQL)
-      # much faster.)
-      cursor = connection.select_cursor(
-        :table => change_log_table,
-        :from => {'id' => current_id},
-        :exclude_starting_row => true,
-        :row_buffer_size => 1
-      )
-      return unless cursor.next?
-
-      # Something is here. Let's actually load it.
-      cursor = connection.select_cursor(
-        :table => change_log_table,
-        :from => {'id' => current_id},
-        :exclude_starting_row => true,
-        :type_cast => true
-      )
-      while cursor.next?
-        change = cursor.next_row
-        self.current_id = change['id']
-        self.change_array << change
-        change['array_index'] = self.change_array.size - 1
-
-        table_change_tree = change_tree[change['change_table']] ||= {}
-        key_changes = table_change_tree[change['change_key']] ||= []
-        key_changes << change
-      end
-      cursor.clear
-    end
-
-    # Returns the creation time of the oldest unprocessed change log record.
-    def oldest_change_time
-      change = oldest_change
-      change['change_time'] if change
-    end
-
-    # Returns the oldest unprocessed change log record (column_name => value hash).
-    def oldest_change
-      update
-      oldest_change = nil
-      unless change_array.empty?
-        while (oldest_change = change_array[self.current_index]) == nil
-          self.current_index += 1
-        end
-      end
-      oldest_change
-    end
-
-    # Returns the specified change log record (column_name => value hash).
-    # * +change_table+: the name of the table that was changed
-    # * +change_key+: the change key of the modified record
-    def load(change_table, change_key)
-      update
-      change = nil
-      table_change_tree = change_tree[change_table]
-      if table_change_tree
-        key_changes = table_change_tree[change_key]
-        if key_changes
-          # get change object and delete from key_changes
-          change = key_changes.shift
-
-          # delete change from change_array
-          change_array[change['array_index']] = nil
-
-          # delete change from database
-          connection.execute "delete from #{change_log_table} where id = #{change['id']}"
-
-          # delete key_changes if empty
-          if key_changes.empty?
-            table_change_tree.delete change_key
-          end
-
-          # delete table_change_tree if empty
-          if table_change_tree.empty?
-            change_tree.delete change_table
-          end
-
-          # reset everything if no more changes remain
-          if change_tree.empty?
-            init_cache
-          end
-        end
-      end
-      change
-    end
-
-    # Returns the name of the change log table
-    def change_log_table
-      @change_log_table ||= "#{session.configuration.options[:rep_prefix]}_pending_changes"
-    end
-    private :change_log_table
-  end
-
   # Describes a single logged record change.
   # 
   # Note:
@@ -187,11 +8,18 @@ module RR
   # Also at the end of change processing the transaction must be committed.
   class LoggedChange
 
+    # The current LoggedChangeLoader
+    attr_accessor :loader
+
     # The current Session
-    attr_accessor :session
+    def session
+      @session ||= loader.session
+    end
 
-    # The database which was changed. Either :+left+ or :+right+.
-    attr_accessor :database
+    # The current database (either +:left+ or +:right+)
+    def database
+      @database ||= loader.database
+    end
 
     # The name of the changed table
     attr_accessor :table
@@ -213,11 +41,10 @@ module RR
     attr_accessor :new_key
 
     # Creates a new LoggedChange instance.
-    # * +session+: the current Session
+    # * +loader+: the current LoggedChangeLoader
     # * +database+: either :+left+ or :+right+
-    def initialize(session, database)
-      self.session = session
-      self.database = database
+    def initialize(loader)
+      self.loader = loader
       self.type = :no_change
     end
 
@@ -281,7 +108,7 @@ module RR
       end.join(key_sep)
       current_key = org_key
 
-      while change = session.change_loader(database).load(table, current_key)
+      while change = loader.load(table, current_key)
 
         new_type = change['change_type']
         current_type = TYPE_CHANGES["#{current_type}#{new_type}"]
@@ -313,16 +140,10 @@ module RR
       load
     end
 
-    # Returns the time of the oldest change. Returns +nil+ if there are no
-    # changes left.
-    def oldest_change_time
-      session.change_loader(database).oldest_change_time
-    end
-
     # Loads the oldest available change
     def load_oldest
       begin
-        change = session.change_loader(database).oldest_change
+        change = loader.oldest_change
         break unless change
         self.key = key_to_hash(change['change_key'])
         self.table = change['change_table']
@@ -332,7 +153,7 @@ module RR
 
     # Prevents session from going into YAML output
     def to_yaml_properties
-      instance_variables.sort.reject {|var_name| var_name == '@session'}
+      instance_variables.sort.reject {|var_name| ['@session', '@loader'].include? var_name}
     end
 
   end

data/lib/rubyrep/logged_change_loader.rb

@@ -0,0 +1,196 @@
+module RR
+  
+  # Makes management of logged change loaders easier
+  class LoggedChangeLoaders
+
+    # The current Session
+    attr_accessor :session
+
+    # A hash of LoggedChangeLoader instances for the :+left+ and :+right+ database
+    attr_accessor :loaders
+
+    # Create new logged change loaders.
+    # * +session+: Current Session
+    def initialize(session)
+      self.session = session
+      self.loaders = {}
+      [:left, :right].each do |database|
+        loaders[database] = LoggedChangeLoader.new(session, database)
+      end
+    end
+
+    # Returns the LoggedChangeLoader for the specified (:+left+ or :+right+)
+    # database.
+    def [](database)
+      loaders[database]
+    end
+
+    # Forces an update of the change log cache
+    def update
+      [:left, :right].each {|database| self[database].update :forced => true}
+    end
+  end
+
+  # Caches the entries in the change log table
+  class LoggedChangeLoader
+
+    # The current +Session+.
+    attr_accessor :session
+
+    # Current database (either :+left+ or :+right+)
+    attr_accessor :database
+
+    # The current +ProxyConnection+.
+    attr_accessor :connection
+
+    # Index to the next unprocessed change in the +change_array+.
+    attr_accessor :current_index
+
+    # ID of the last cached change log record.
+    attr_accessor :current_id
+
+    # Array with all cached changes.
+    # Processed change log records are replaced with +nil+.
+    attr_accessor :change_array
+
+    # Tree (hash) structure for fast access to all cached changes.
+    # First level of tree:
+    # * key: table name
+    # * value: 2nd level tree
+    # 2nd level tree:
+    # * key: the change_key value of the according change log records.
+    # * value:
+    #   An array of according change log records (column_name => value hash).
+    #   Additional entry of each change log hash:
+    #   * key: 'array_index'
+    #   * value: index to the change log record in +change_array+
+    attr_accessor :change_tree
+
+    # Date of last update of the cache
+    attr_accessor :last_updated
+
+    # Initializes / resets the cache.
+    def init_cache
+      self.change_tree = {}
+      self.change_array = []
+      self.current_index = 0
+    end
+    private :init_cache
+
+    # Create a new change log record cache.
+    # * +session+: The current +Session+
+    # * +database+: Either :+left+ or :+right+
+    def initialize(session, database)
+      self.session = session
+      self.database = database
+      self.connection = session.send(database)
+
+      init_cache
+      self.current_id = -1
+      self.last_updated = 1.year.ago
+    end
+
+    # Updates the cache.
+    # Options is a hash determining when the update is actually executed:
+    # * :+expire_time+: cache is older than the given number of seconds
+    # * :+forced+: if +true+ update the cache even if not yet expired
+    def update(options = {:forced => false, :expire_time => 1})
+      return unless options[:forced] or Time.now - self.last_updated >= options[:expire_time]
+      
+      self.last_updated = Time.now
+
+      # First, let's use a LIMIT clause (via :row_buffer_size option) to verify
+      # if there are any pending changes.
+      # (If there are many pending changes, this is (at least with PostgreSQL)
+      # much faster.)
+      cursor = connection.select_cursor(
+        :table => change_log_table,
+        :from => {'id' => current_id},
+        :exclude_starting_row => true,
+        :row_buffer_size => 1
+      )
+      return unless cursor.next?
+
+      # Something is here. Let's actually load it.
+      cursor = connection.select_cursor(
+        :table => change_log_table,
+        :from => {'id' => current_id},
+        :exclude_starting_row => true,
+        :type_cast => true
+      )
+      while cursor.next?
+        change = cursor.next_row
+        self.current_id = change['id']
+        self.change_array << change
+        change['array_index'] = self.change_array.size - 1
+
+        table_change_tree = change_tree[change['change_table']] ||= {}
+        key_changes = table_change_tree[change['change_key']] ||= []
+        key_changes << change
+      end
+      cursor.clear
+    end
+
+    # Returns the creation time of the oldest unprocessed change log record.
+    def oldest_change_time
+      change = oldest_change
+      change['change_time'] if change
+    end
+
+    # Returns the oldest unprocessed change log record (column_name => value hash).
+    def oldest_change
+      update
+      oldest_change = nil
+      unless change_array.empty?
+        while (oldest_change = change_array[self.current_index]) == nil
+          self.current_index += 1
+        end
+      end
+      oldest_change
+    end
+
+    # Returns the specified change log record (column_name => value hash).
+    # * +change_table+: the name of the table that was changed
+    # * +change_key+: the change key of the modified record
+    def load(change_table, change_key)
+      update
+      change = nil
+      table_change_tree = change_tree[change_table]
+      if table_change_tree
+        key_changes = table_change_tree[change_key]
+        if key_changes
+          # get change object and delete from key_changes
+          change = key_changes.shift
+
+          # delete change from change_array
+          change_array[change['array_index']] = nil
+
+          # delete change from database
+          connection.execute "delete from #{change_log_table} where id = #{change['id']}"
+
+          # delete key_changes if empty
+          if key_changes.empty?
+            table_change_tree.delete change_key
+          end
+
+          # delete table_change_tree if empty
+          if table_change_tree.empty?
+            change_tree.delete change_table
+          end
+
+          # reset everything if no more changes remain
+          if change_tree.empty?
+            init_cache
+          end
+        end
+      end
+      change
+    end
+
+    # Returns the name of the change log table
+    def change_log_table
+      @change_log_table ||= "#{session.configuration.options[:rep_prefix]}_pending_changes"
+    end
+    private :change_log_table
+  end
+end

data/lib/rubyrep/noisy_connection.rb

@@ -0,0 +1,80 @@
+module RR
+
+  # Wraps an existing cursor.
+  # Purpose: send regular updates to the installed TaskSweeper
+  class NoisyCursor
+    # The original cusor
+    attr_accessor :org_cursor
+
+    # The installed task sweeper
+    attr_accessor :sweeper
+
+    # Create a new NoisyCursor.
+    # * cursor: the original cursor
+    # * sweeper: the target TaskSweeper
+    def initialize(cursor, sweeper)
+      self.org_cursor = cursor
+      self.sweeper = sweeper
+    end
+
+    # Delegate the uninteresting methods to the original cursor
+    def next?; org_cursor.next? end
+    def clear; org_cursor.clear end
+
+    # Returns the row as a column => value hash and moves the cursor to the next row.
+    def next_row
+      sweeper.ping
+      row = org_cursor.next_row
+      sweeper.ping
+      row
+    end
+  end
+
+  # Modifies ProxyConnections to send regular pings to an installed TaskSweeper
+  module NoisyConnection
+
+    # The installed TaskSweeper
+    attr_accessor :sweeper
+
+    # Modifies ProxyConnection#select_cursor to wrap the returned cursor
+    # into a NoisyCursor.
+    def select_cursor(options)
+      sweeper.ping
+      org_cursor = super
+      sweeper.ping
+      NoisyCursor.new(org_cursor, sweeper)
+    end
+
+    # Wraps ProxyConnection#insert_record to update the TaskSweeper
+    def insert_record(table, values)
+      sweeper.ping
+      result = super
+      sweeper.ping
+      result
+    end
+
+    # Wraps ProxyConnection#update_record to update the TaskSweeper
+    def update_record(table, values, org_key = nil)
+      sweeper.ping
+      result = super
+      sweeper.ping
+      result
+    end
+
+    # Wraps ProxyConnection#delete_record to update the TaskSweeper
+    def delete_record(table, values)
+      sweeper.ping
+      result = super
+      sweeper.ping
+      result
+    end
+
+    # Wraps ProxyConnection#commit_db_transaction to update the TaskSweeper
+    def commit_db_transaction
+      sweeper.ping
+      result = super
+      sweeper.ping
+      result
+    end
+  end
+end

data/lib/rubyrep/proxy_connection.rb

@@ -198,21 +198,14 @@ module RR
       self.manual_primary_keys = {}
     end
 
-    # Checks if the connection is still active and if not, reestablished it.
-    def refresh
-      unless self.connection.active?
-        self.connection = ConnectionExtenders.db_connect config
-      end
-    end
-    
     # Destroys the session
     def destroy
-      self.connection.disconnect!
-      
       cursors.each_key do |cursor|
         cursor.destroy
       end
       cursors.clear
+
+      self.connection.disconnect!
     end
     
     # Quotes the given value. It is assumed that the value belongs to the specified column name and table name.

data/lib/rubyrep/replication_difference.rb

@@ -5,7 +5,12 @@ module RR
   class ReplicationDifference
 
     # The current Session.
-    attr_accessor :session
+    def session
+      @session ||= loaders.session
+    end
+
+    # The current LoggedChangeLoaders instance
+    attr_accessor :loaders
     
     # The type of the difference. Either
     # * :+left+: change in left database
@@ -21,9 +26,9 @@ module RR
     end
 
     # Creates a new ReplicationDifference instance.
-    # +session+ is the current Session.
-    def initialize(session)
-      self.session = session
+    # +loaders+ is teh current LoggedChangeLoaders instance
+    def initialize(loaders)
+      self.loaders = loaders
     end
 
     # Should be set to +true+ if this ReplicationDifference instance was
@@ -52,7 +57,7 @@ module RR
 
     # Amends a difference according to new entries in the change log table
     def amend
-      session.reload_changes
+      loaders.update
       changes[:left].load
       changes[:right].load
       self.type = DIFF_TYPES[changes[:left].type][changes[:right].type]
@@ -62,8 +67,8 @@ module RR
     def load
       change_times = {}
       [:left, :right].each do |database|
-        changes[database] = LoggedChange.new session, database
-        change_times[database] = changes[database].oldest_change_time
+        changes[database] = LoggedChange.new loaders[database]
+        change_times[database] = loaders[database].oldest_change_time
       end
       return if change_times[:left] == nil and change_times[:right] == nil
 
@@ -82,9 +87,9 @@ module RR
       self.loaded = true
     end
 
-    # Prevents session from going into YAML output
+    # Prevents session and change loaders from going into YAML output
     def to_yaml_properties
-      instance_variables.sort.reject {|var_name| var_name == '@session'}
+      instance_variables.sort.reject {|var_name| ['@session', '@loaders'].include? var_name}
     end
 
   end

data/lib/rubyrep/replication_initializer.rb

@@ -262,6 +262,13 @@ module RR
       end
     end
 
+    # Calls the potentially provided :+after_init+ handler after infrastructure
+    # tables are created.
+    def call_after_infrastructure_setup_handler
+      handler = session.configuration.options[:after_infrastructure_setup]
+      handler.call(session) if handler
+    end
+
     # Prepares the database / tables for replication.
     def prepare_replication
       exclude_rubyrep_tables
@@ -269,6 +276,8 @@ module RR
       puts "Verifying RubyRep tables"
       ensure_infrastructure
 
+      call_after_infrastructure_setup_handler
+
       puts "Checking for and removing rubyrep triggers from unconfigured tables"
       restore_unconfigured_tables