rubyrep 1.0.9 → 1.1.0

data/History.txt

@@ -1,3 +1,8 @@
+== 1.1.0 2009-12-15
+
+* Feature: filtering of sync / replication events with custom specified functionality
+* Feature: compatibility with JRuby 1.4.0 and ActiveRecord 2.3.5
+
 == 1.0.9 2009-10-14
 
 * Bug fix: ReplicationRun#run should load only one change log record to figure out if there are pending changes

data/config/hoe.rb

@@ -59,8 +59,8 @@ hoe = Hoe.new(GEM_NAME, VERS) do |p|
   p.changes = p.paragraphs_of("History.txt", 0..1).join("\\n\\n")
   #p.extra_deps = []     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
   p.extra_deps = [
-    ['activesupport', '>= 2.2.2'],
-    ['activerecord' , '>= 2.2.2']
+    ['activesupport', '>= 2.3.5'],
+    ['activerecord' , '>= 2.3.5']
   ]
   
   #p.spec_extras = {}    # A hash of extra values to set in the gemspec.

data/config/requirements.rb

@@ -4,7 +4,7 @@ include FileUtils
 require 'rubygems'
 
 # Load essential gems. Exit if not available.
-%w[rake activerecord].each do |req_gem|
+%w[rake active_record].each do |req_gem|
   begin
     require req_gem
   rescue LoadError

data/lib/rubyrep/committers/buffered_committer.rb

@@ -16,11 +16,13 @@ module RR
 
       # Switches the trigger mode of the specified +table+ in the specified
       # +database+ to ignore rubyrep activity.
+      # * +database+: identifying the database (either :+left+ or :+right+)
+      # * +table+: name of the table
       def exclude_rr_activity(database, table)
         trigger_mode_switcher.exclude_rr_activity database, table
       end
 
-      # Returns the trigger mode switcher (creates it if necessary)
+      # Returns the TriggerModeSwitcher (creates it if necessary)
       def trigger_mode_switcher
         @trigger_mode_switcher ||= TriggerModeSwitcher.new session
       end
@@ -96,36 +98,39 @@ module RR
       end
 
       # A new committer is created for each table sync.
-      #   * session: a Session object representing the current database session
+      # * session: a Session object representing the current database session
       def initialize(session)
         super
         begin_db_transactions
       end
 
-      # Inserts the specified record in the specified +database+ (either :left or :right).
-      # +table+ is the name of the target table.
-      # +values+ is a hash of column_name => value pairs.
+      # Inserts the specified record in the specified database.
+      # * +database+: identifying the database (either :+left+ or :+right+)
+      # * +table+: name of the table
+      # * +values+: a hash of column_name => value pairs.
       def insert_record(database, table, values)
         exclude_rr_activity database, table
         super
         commit
       end
 
-      # Updates the specified record in the specified +database+ (either :left or :right).
-      # +table+ is the name of the target table.
-      # +values+ is a hash of column_name => value pairs.
-      # +old_key+ is a column_name => value hash with the original primary key.
-      # If +old_key+ is +nil+, then the primary key must be contained in +values+.
+      # Updates the specified record in the specified database.
+      # * +database+: identifying the database (either :+left+ or :+right+)
+      # * +table+: name of the table
+      # * +values+: a hash of column_name => value pairs.
+      # * +old_key+:
+      #   A column_name => value hash identifying original primary key.
+      #   If +nil+, then the primary key must be contained in +values+.
       def update_record(database, table, values, old_key = nil)
         exclude_rr_activity database, table
         super
         commit
       end
 
-      # Deletes the specified record in the specified +database+ (either :left or :right).
-      # +table+ is the name of the target table.
-      # +values+ is a hash of column_name => value pairs. (Only the primary key
-      # values will be used and must be included in the hash.)
+      # Deletes the specified record in the specified database.
+      # * +database+: identifying the database (either :+left+ or :+right+)
+      # * +table+: name of the table
+      # * +values+: a hash of column_name => value pairs (must only contain primary key columns).
       def delete_record(database, table, values)
         exclude_rr_activity database, table
         super
@@ -133,7 +138,7 @@ module RR
       end
 
       # Is called after the last insert / update / delete query.
-      # +success+ should be true if there were no problems, false otherwise.
+      # * +success+: should be true if there were no problems, false otherwise.
       def finalize(success = true)
         if success
           commit_db_transactions

data/lib/rubyrep/configuration.rb

@@ -239,14 +239,18 @@ module RR
         resulting_options.merge! table_options[1] if match
       end
 
-      # Merge the default syncer options in (if syncer has some)
-      syncer_class = Syncers.configured_syncer(resulting_options)
-      if syncer_class.respond_to? :default_options
-        default_syncer_options = syncer_class.default_options.clone
-      else
-        default_syncer_options = {}
+      # Merge the default syncer& replicator options in
+      [
+        Syncers.configured_syncer(resulting_options),
+        Replicators.configured_replicator(resulting_options)
+      ].each do |processor_class|
+        if processor_class.respond_to? :default_options
+          default_processor_options = processor_class.default_options.clone
+        else
+          default_processor_options = {}
+        end
+        resulting_options = default_processor_options.merge!(resulting_options)
       end
-      resulting_options = default_syncer_options.merge! resulting_options
 
       resulting_options
     end

data/lib/rubyrep/connection_extenders/connection_extenders.rb

@@ -76,7 +76,7 @@ module RR
       # Hack to get Postgres schema support under JRuby to par with the standard
       # ruby version
       if RUBY_PLATFORM =~ /java/ and config[:adapter].to_sym == :postgresql
-        connection.extend RR::ConnectionExtenders::JdbcPostgreSQLExtender
+        connection.extend RR::ConnectionExtenders::PostgreSQLExtender
         connection.initialize_search_path
       end
 

data/lib/rubyrep/connection_extenders/jdbc_extender.rb

@@ -55,44 +55,26 @@ module RR
         result
       end
     end
+  end
+end
 
-    # PostgreSQL specific functionality not provided by the standard JDBC
-    # connection extender:
-    # * Hack to get schema support for Postgres under JRuby on par with the
-    #   standard ruby version.
-    module JdbcPostgreSQLExtender
-      require 'jdbc_adapter/jdbc_postgre'
-      require "#{File.dirname(__FILE__)}/postgresql_extender"
-
-      include PostgreSQLExtender
-
-      class JdbcPostgreSQLColumn < ActiveRecord::ConnectionAdapters::Column
-        include ::JdbcSpec::PostgreSQL::Column
-      end
-
-      # Returns the list of all column definitions for a table.
-      def columns(table_name, name = nil)
-        # Limit, precision, and scale are all handled by the superclass.
-        column_definitions(table_name).collect do |name, type, default, notnull|
-          JdbcPostgreSQLColumn.new(name, default, type, notnull == 'f')
-        end
-      end
-
-      # Sets the schema search path as per configuration parameters
-      def initialize_search_path
-        execute "SET search_path TO #{config[:schema_search_path]}" if config[:schema_search_path]
+# activerecord-jdbc-adapter 0.9.1 (7b3f3eca08149567070837fad63696052dc36cd6)
+# improves SQLite binary support by overwriting the global string_to_binary
+# methods.
+# This appears to break binary support for MySQL.
+# And here comes the monkey patch to revert it again...
+require 'active_record/connection_adapters/jdbc_adapter_spec'
+require 'jdbc_adapter/jdbc_sqlite3'
+module ::ActiveRecord
+  module ConnectionAdapters
+    class JdbcColumn < Column
+      def self.string_to_binary(value)
+        value
       end
 
-      # Converts the given Time object into the correctly formatted string
-      # representation.
-      # 
-      # Monkeypatched as activerecord-jdbcpostgresql-adapter (at least in version
-      # 0.8.2) does otherwise "loose" the microseconds when writing Time values
-      # to the database.
-      def quoted_date(value)
-        "#{value.strftime("%Y-%m-%d %H:%M:%S")}#{value.respond_to?(:usec) ? ".#{value.usec.to_s.rjust(6, '0')}" : ""}"
+      def self.binary_to_string(value)
+        value
       end
     end
   end
 end
-

data/lib/rubyrep/connection_extenders/postgresql_extender.rb

@@ -107,6 +107,15 @@ module RR
         SQL
       end
 
+      # Disables schema extraction from table names by overwriting the according
+      # ActiveRecord method.
+      # Necessary to support table names containing dots (".").
+      # (This is possible as rubyrep exclusively uses the search_path setting to
+      # support PostgreSQL schemas.)
+      def extract_pg_identifier_from_name(name)
+        return name, nil
+      end
+
       # Returns an ordered list of primary key column names of the given table
       def primary_key_names(table)
         row = self.select_one(<<-end_sql)
@@ -177,6 +186,63 @@ module RR
         result
       end
 
+      # Sets the schema search path as per configuration parameters
+      def initialize_search_path
+        execute "SET search_path TO #{config[:schema_search_path] || 'public'}"
+      end
+
+      # *** Moneky patch***
+      # Returns the column objects for the named table.
+      # Fixes JRuby schema support
+      def columns(table_name, name = nil)
+        jdbc_connection = @connection.connection # the actual JDBC DatabaseConnection
+        @unquoted_schema ||= select_one("show search_path")['search_path']
+
+        # check if table exists
+        table_results = jdbc_connection.meta_data.get_tables(
+          jdbc_connection.catalog,
+          @unquoted_schema,
+          table_name,
+          ["TABLE","VIEW","SYNONYM"].to_java(:string)
+        )
+        table_exists = table_results.next
+        table_results.close
+        raise "table '#{table_name}' not found" unless table_exists
+
+        # get ResultSet for columns of table
+        column_results = jdbc_connection.meta_data.get_columns(
+          jdbc_connection.catalog,
+          @unquoted_schema,
+          table_name,
+          nil
+        )
+        
+        # create the Column objects
+        columns = []
+        while column_results.next
+          
+          # generate type clause
+          type_clause = column_results.get_string('TYPE_NAME')
+          precision = column_results.get_int('COLUMN_SIZE')
+          scale = column_results.get_int('DECIMAL_DIGITS')
+          if precision > 0
+            type_clause += "(#{precision}#{scale > 0 ? ",#{scale}" : ""})"
+          end
+
+          # create column
+          columns << ::ActiveRecord::ConnectionAdapters::JdbcColumn.new(
+            @config,
+            column_results.get_string('COLUMN_NAME'),
+            column_results.get_string('COLUMN_DEF'),
+            type_clause,
+            column_results.get_string('IS_NULLABLE').strip == "NO"
+          )
+        end
+        column_results.close
+        
+        columns
+      end if RUBY_PLATFORM =~ /java/
+
       # *** Monkey patch***
       # Returns the list of a table's column names, data types, and default values.
       # This overwrites the according ActiveRecord::PostgreSQLAdapter method

data/lib/rubyrep/proxy_connection.rb

@@ -184,19 +184,46 @@ module RR
     # * first build the query based on options forwarded to #table_select_query
     # +options+ is a hash with
     # * :+query+: executes the given query
-    # * :+type_cast+: if +true+, build a type casting cursor around the result
-    # * :+table+: name of the table from which to read data
+    # * :+type_cast+:
+    #   Unless explicitely disabled with +false+, build type casting cursor
+    #   around result.
+    # * :+table+:
+    #   Name of the table from which to read data.
+    #   Required unless type casting is disabled.
     # * further options as taken by #table_select_query to build the query
     # * :+row_buffer_size+:
     #   Integer controlling how many rows a read into memory at one time.
     def select_cursor(options)
       cursor = ResultFetcher.new(self, options)
-      if options[:type_cast]
+      unless options[:type_cast] == false
         cursor = TypeCastingCursor.new(self, options[:table], cursor)
       end
       cursor
     end
+
+    # Reads the designated record from the database.
+    # Refer to #select_cursor for details parameter description.
+    # Returns the first matching row (column_name => value hash or +nil+).
+    def select_record(options)
+      cursor = select_cursor({:row_buffer_size => 1}.merge(options))
+      row = cursor.next? ? cursor.next_row : nil
+      cursor.clear
+      row
+    end
     
+    # Reads the designated records from the database.
+    # Refer to #select_cursor for details parameter description.
+    # Returns an array of matching rows (column_name => value hashes).
+    def select_records(options)
+      cursor = select_cursor(options)
+      rows = []
+      while cursor.next?
+        rows << cursor.next_row
+      end
+      cursor.clear
+      rows
+    end
+
     # Create a session on the proxy side according to provided configuration hash.
     # +config+ is a hash as described by ActiveRecord::Base#establish_connection
     def initialize(config)

data/lib/rubyrep/replication_helper.rb

@@ -16,6 +16,16 @@ module RR
     # Current options
     def options; @options ||= session.configuration.options; end
 
+    # Returns the options for the specified table name.
+    # * +table+: name of the table (left database version)
+    def options_for_table(table)
+      @options_for_table ||= {}
+      unless @options_for_table.include? table
+        @options_for_table[table] = session.configuration.options_for_table(table)
+      end
+      @options_for_table[table]
+    end
+
     # Delegates to Session#corresponding_table
     def corresponding_table(db_arm, table); session.corresponding_table(db_arm, table); end
 
@@ -25,17 +35,17 @@ module RR
       committer.new_transaction?
     end
 
-    # Delegates to Committer#insert_record
+    # Delegates to Committers::BufferedCommitter#insert_record
     def insert_record(database, table, values)
       committer.insert_record(database, table, values)
     end
 
-    # Delegates to Committer#insert_record
+    # Delegates to Committers::BufferedCommitter#update_record
     def update_record(database, table, values, old_key = nil)
       committer.update_record(database, table, values, old_key)
     end
 
-    # Delegates to Committer#insert_record
+    # Delegates to Committers::BufferedCommitter#delete_record
     def delete_record(database, table, values)
       committer.delete_record(database, table, values)
     end
@@ -67,6 +77,26 @@ module RR
       committer.finalize(success)
     end
 
+    # Converts the row values into their proper types as per table definition.
+    # * +table+: name of the table after whose columns is type-casted.
+    # * +row+: A column_name => value hash of the row
+    # Returns a copy of the column_name => value hash (with type-casted values).
+    def type_cast(table, row)
+      @table_columns ||= {}
+      unless @table_columns.include?(table)
+        column_array = session.left.columns(table)
+        column_hash = {}
+        column_array.each {|column| column_hash[column.name] = column}
+        @table_columns[table] = column_hash
+      end
+      columns = @table_columns[table]
+      type_casted_row = {}
+      row.each_pair do |column_name, value|
+        type_casted_row[column_name] = columns[column_name].type_cast(value)
+      end
+      type_casted_row
+    end
+
     # Logs the outcome of a replication into the replication log table.
     # * +diff+: the replicated ReplicationDifference
     # * +outcome+: string summarizing the outcome of the replication

data/lib/rubyrep/replication_run.rb

@@ -22,6 +22,23 @@ module RR
         Replicators.replicators[session.configuration.options[:replicator]].new(helper)
     end
 
+    # Calls the event filter for the give  difference.
+    # * +diff+: instance of ReplicationDifference
+    # Returns +true+ if replication of the difference should *not* proceed.
+    def event_filtered?(diff)
+      event_filter = helper.options_for_table(diff.changes[:left].table)[:event_filter]
+      if event_filter && event_filter.respond_to?(:before_replicate)
+        not event_filter.before_replicate(
+          diff.changes[:left].table,
+          helper.type_cast(diff.changes[:left].table, diff.changes[:left].key),
+          helper, 
+          diff
+        )
+      else
+        false
+      end
+    end
+
     # Executes the replication run.
     def run
       return unless [:left, :right].any? do |database|
@@ -53,7 +70,9 @@ module RR
             diff.load
             break unless diff.loaded?
             break if sweeper.terminated?
-            replicator.replicate_difference diff if diff.type != :no_diff
+            if diff.type != :no_diff and not event_filtered?(diff)
+              replicator.replicate_difference diff
+            end
           rescue Exception => e
             begin
               helper.log_replication_outcome diff, e.message,

data/lib/rubyrep/replicators/replicators.rb

@@ -23,7 +23,12 @@ module RR
       @replicators ||= {}
       @replicators
     end
-  
+
+    # Returns the correct replicator class as per provided options hash
+    def self.configured_replicator(options)
+      replicators[options[:replicator]]
+    end
+
     # Registers one or multiple replicators.
     # syncer_hash is a Hash with
     #   key::   The adapter symbol as used to reference the replicator

data/lib/rubyrep/replicators/two_way_replicator.rb

@@ -60,13 +60,16 @@ module RR
       # The current ReplicationHelper object
       attr_accessor :rep_helper
 
-      # Default TwoWayReplicator options.
-      DEFAULT_OPTIONS =  {
-        :left_change_handling => :replicate,
-        :right_change_handling => :replicate,
-        :replication_conflict_handling => :ignore,
-        :logged_replication_events => [:ignored_conflicts],
-      }
+      # Provides default option for the replicator. Optional.
+      # Returns a hash with key => value pairs.
+      def self.default_options
+        {
+          :left_change_handling => :replicate,
+          :right_change_handling => :replicate,
+          :replication_conflict_handling => :ignore,
+          :logged_replication_events => [:ignored_conflicts],
+        }
+      end
 
       # Checks if an option is configured correctly. Raises an ArgumentError if not.
       # * +table_spec+: the table specification to which the option belongs. May be +nil+.
@@ -173,23 +176,12 @@ module RR
         end
       end
 
-      # Returns the options for the specified table name.
-      # * +table+: name of the table (left database version)
-      def options_for_table(table)
-        @options_for_table ||= {}
-        unless @options_for_table.include? table
-          @options_for_table[table] = DEFAULT_OPTIONS.merge(
-            rep_helper.session.configuration.options_for_table(table))
-        end
-        @options_for_table[table]
-      end
-
       # Logs replication of the specified difference as per configured
       # :+replication_conflict_logging+ / :+left_change_logging+ / :+right_change_logging+ options.
       # * +winner+: Either the winner database (:+left+ or :+right+) or :+ignore+
       # * +diff+: the ReplicationDifference instance
       def log_replication_outcome(winner, diff)
-        options = options_for_table(diff.changes[:left].table)
+        options = rep_helper.options_for_table(diff.changes[:left].table)
         option_values = [options[:logged_replication_events]].flatten # make sure I have an array
         if diff.type == :conflict
           return unless option_values.include?(:all_conflicts) or option_values.include?(:ignored_conflicts)
@@ -321,7 +313,7 @@ module RR
       # * :+previous_failure_description+: why the previous replication attempt failed
       def replicate_difference(diff, remaining_attempts = MAX_REPLICATION_ATTEMPTS, previous_failure_description = nil)
         raise Exception, previous_failure_description || "max replication attempts exceeded" if remaining_attempts == 0
-        options = options_for_table(diff.changes[:left].table)
+        options = rep_helper.options_for_table(diff.changes[:left].table)
         if diff.type == :left or diff.type == :right
           key = diff.type == :left ? :left_change_handling : :right_change_handling
           option = options[key]

data/lib/rubyrep/sync_helper.rb

@@ -26,21 +26,29 @@ module RR
       @tables ||= {:left => left_table, :right => right_table}
     end
 
+    # Given a column_name => value hash of a full row, returns a
+    # column_name => value hash of the primary key columns.
+    # * +row+: the full row
+    # Returns
+    def extract_key(row)
+      row.reject {|column, value| not primary_key_names.include? column }
+    end
+
     # Sync options for the current table sync
     def sync_options; @sync_options ||= table_sync.sync_options; end
 
-    # Delegates to Committer#insert_record
-    def insert_record(database, values)
+    # Delegates to Committers::BufferedCommitter#insert_record
+    def insert_record(database, table, values)
       committer.insert_record(database, tables[database], values)
     end
 
-    # Delegates to Committer#insert_record
-    def update_record(database, values, old_key = nil)
+    # Delegates to Committers::BufferedCommitter#update_record
+    def update_record(database, table, values, old_key = nil)
       committer.update_record(database, tables[database], values, old_key)
     end
 
-    # Delegates to Committer#insert_record
-    def delete_record(database, values)
+    # Delegates to Committers::BufferedCommitter#delete_record
+    def delete_record(database, table, values)
       committer.delete_record(database, tables[database], values)
     end
 

data/lib/rubyrep/syncers/syncers.rb

@@ -94,15 +94,15 @@ module RR
         case type
         when source
           if sync_helper.sync_options[:insert]
-            sync_helper.insert_record target, row
+            sync_helper.insert_record target, sync_helper.tables[target], row
           end
         when target
           if sync_helper.sync_options[:delete]
-            sync_helper.delete_record target, row
+            sync_helper.delete_record target, sync_helper.tables[target], row
           end
         when :conflict
           if sync_helper.sync_options[:update]
-            sync_helper.update_record target, row[source_record_index]
+            sync_helper.update_record target, sync_helper.tables[target], row[source_record_index]
           end
         end
       end

data/lib/rubyrep/syncers/two_way_syncer.rb

@@ -148,10 +148,10 @@ module RR
           if option == :ignore
             # nothing to do
           elsif option == :delete
-            sync_helper.delete_record type, row
+            sync_helper.delete_record type, sync_helper.tables[type], row
           elsif option == :insert
             target = (type == :left ? :right : :left)
-            sync_helper.insert_record target, row
+            sync_helper.insert_record target, sync_helper.tables[target], row
           else #option must be a Proc
             option.call sync_helper, type, row
           end
@@ -161,9 +161,9 @@ module RR
           if option == :ignore
             # nothing to do
           elsif option == :right_wins
-            sync_helper.update_record :left, row[1]
+            sync_helper.update_record :left, sync_helper.tables[:left], row[1]
           elsif option == :left_wins
-            sync_helper.update_record :right, row[0]
+            sync_helper.update_record :right, sync_helper.tables[:right], row[0]
           else #option must be a Proc
             option.call sync_helper, type, row
           end

data/lib/rubyrep/table_sync.rb

@@ -34,6 +34,26 @@ module RR
       end
     end
 
+    # Calls the event filter for the give table difference.
+    # * +type+: type of difference
+    # * +row+: the differing row
+    # Refer to DirectTableScan#run for full description of +type+ and +row+.
+    # Returns +true+ if syncing of the difference should *not* proceed.
+    def event_filtered?(type, row)
+      event_filter = sync_options[:event_filter]
+      if event_filter && event_filter.respond_to?(:before_sync)
+        not event_filter.before_sync(
+          helper.left_table,
+          helper.extract_key([row].flatten.first),
+          helper,
+          type,
+          row
+        )
+      else
+        false
+      end
+    end
+
     # Executes the table sync. If a block is given, yields each difference with
     # the following 2 parameters
     # * +type+
@@ -54,7 +74,9 @@ module RR
 
       scan.run do |type, row|
         yield type, row if block_given? # To enable progress reporting
-        syncer.sync_difference type, row
+        unless event_filtered?(type, row)
+          syncer.sync_difference type, row
+        end
       end
       
       execute_sync_hook :after_table_sync

data/lib/rubyrep/version.rb

@@ -1,8 +1,8 @@
 module RR #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 1
-    MINOR = 0
-    TINY  = 9
+    MINOR = 1
+    TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end