rubyrep 1.1.2 → 1.2.0

data/History.txt

@@ -1,3 +1,12 @@
+== 1.2.0 2011-03-07
+
+* Feature: compatibility with Rails 3
+* Feature: do not replicate record updates that didn't change any fields (props to daudo)
+* Bug fix: reducing deadlock problems (props to gtanzillo)
+* Bug fix: adding missing schema prefix in PostgreSQL triggers
+* Bug fix: scans / syncs fail due to incorrect handling of case sensitivity of string primary key columns
+* Bug fix: reducing risk of foreign key conflicts during replication (props for root cause analysis to TonyB)
+
 == 1.1.2 2009-05-10
 
 * Bug fix: escape primary keys in replication triggers

data/config/hoe.rb

@@ -45,25 +45,23 @@ end
 
 # Generate all the Rake tasks
 # Run 'rake -T' to see list of generated tasks (from gem root directory)
-hoe = Hoe.new(GEM_NAME, VERS) do |p|
-  p.author = AUTHOR 
-  p.description = DESCRIPTION
-  p.email = EMAIL
-  p.summary = DESCRIPTION
-  p.url = HOMEPATH
-  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
-  p.test_globs = ["test/**/test_*.rb"]
-  p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']  #An array of file patterns to delete on clean.
-  
+hoe = Hoe.spec(GEM_NAME) do
+  self.version = VERS
+  developer AUTHOR, EMAIL
+  description = DESCRIPTION
+  summary = DESCRIPTION
+  url = HOMEPATH
+  rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  test_globs = ["test/**/test_*.rb"]
+  clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']  #An array of file patterns to delete on clean.
+
   # == Optional
-  p.changes = p.paragraphs_of("History.txt", 0..1).join("\\n\\n")
-  #p.extra_deps = []     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
-  p.extra_deps = [
-    ['activesupport', '>= 2.3.5'],
-    ['activerecord' , '>= 2.3.5']
-  ]
+  changes = paragraphs_of("History.txt", 0..1).join("\\n\\n")
+  #extra_deps = []     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
+  extra_deps << ['activesupport', '>= 3.0.5']
+  extra_deps << ['activerecord' , '>= 3.0.5']
   
-  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
+  #spec_extras = {}    # A hash of extra values to set in the gemspec.
   
 end
 

data/lib/rubyrep.rb

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift File.dirname(__FILE__) + "/rubyrep"
 require 'rubygems'
 require 'yaml'
 
-gem 'activerecord', '>= 2.2.2'
+gem 'activerecord', '>= 3.0.5'
 require 'active_record'
 
 require 'version'

data/lib/rubyrep/connection_extenders/connection_extenders.rb

@@ -1,3 +1,8 @@
+class ActiveRecord::ConnectionAdapters::AbstractAdapter
+  # The current log subscriber
+  attr_accessor :log_subscriber
+end
+
 class ActiveRecord::ConnectionAdapters::Column
   # Bug in ActiveRecord parsing of PostgreSQL timestamps with microseconds:
   # Certain values are incorrectly rounded, thus ending up with timestamps
@@ -107,9 +112,17 @@ module RR
         if config[:logger].respond_to?(:debug)
           logger = config[:logger]
         else
-          logger = Logger.new(config[:logger])
+          logger = ActiveSupport::BufferedLogger.new(config[:logger])
         end
         db_connection.instance_variable_set :@logger, logger
+        if ActiveSupport.const_defined?(:Notifications)
+          connection_object_id = db_connection.object_id
+          db_connection.log_subscriber = ActiveSupport::Notifications.subscribe("sql.active_record") do |name, start, finish, id, payload|
+            if payload[:connection_id] == connection_object_id and logger.debug?
+              logger.debug payload[:sql].squeeze(" ")
+            end
+          end
+        end
       end
     end
     

data/lib/rubyrep/connection_extenders/jdbc_extender.rb

@@ -58,23 +58,8 @@ module RR
   end
 end
 
-# activerecord-jdbc-adapter 0.9.1 (7b3f3eca08149567070837fad63696052dc36cd6)
-# improves SQLite binary support by overwriting the global string_to_binary
-# methods.
-# This appears to break binary support for MySQL.
-# And here comes the monkey patch to revert it again...
-require 'active_record/connection_adapters/jdbc_adapter_spec'
-require 'jdbc_adapter/jdbc_sqlite3'
-module ::ActiveRecord
-  module ConnectionAdapters
-    class JdbcColumn < Column
-      def self.string_to_binary(value)
-        value
-      end
-
-      def self.binary_to_string(value)
-        value
-      end
-    end
-  end
+require 'activerecord-jdbc-adapter'
+if ArJdbc.const_defined?(:PostgreSQL)
+  ArJdbc::PostgreSQL::RecordNotUnique = ActiveRecord::RecordNotUnique unless ArJdbc::PostgreSQL.const_defined?(:RecordNotUnique)
+  ArJdbc::PostgreSQL::InvalidForeignKey = ActiveRecord::InvalidForeignKey  unless ArJdbc::PostgreSQL.const_defined?(:InvalidForeignKey)
 end

data/lib/rubyrep/proxy_connection.rb

@@ -239,6 +239,11 @@ module RR
       end
       cursors.clear
 
+      if connection.log_subscriber
+        ActiveSupport::Notifications.notifier.unsubscribe connection.log_subscriber
+        connection.log_subscriber = nil
+      end
+
       self.connection.disconnect!
     end
     

data/lib/rubyrep/replication_difference.rb

@@ -19,6 +19,10 @@ module RR
     # * :+no_diff+: changes in both databases constitute no difference
     attr_accessor :type
 
+    # Is set to +true+ if first replication attempt failed but it should be tried again later
+    attr_accessor :second_chance
+    alias_method :second_chance?, :second_chance
+
     # A hash with keys :+left+ and / or :+right+.
     # Hash values are LoggedChange instances.
     def changes

data/lib/rubyrep/replication_extenders/postgresql_replication.rb

@@ -45,13 +45,25 @@ module RR
         activity_check = ""
         if params[:exclude_rr_activity] then
           activity_check = <<-end_sql
-            PERFORM ACTIVE FROM #{params[:activity_table]};
+            PERFORM ACTIVE FROM #{schema_prefix}#{params[:activity_table]};
             IF FOUND THEN
               RETURN NULL;
             END IF;
           end_sql
         end
 
+        version_string = select_value("select version();")
+        version = version_string.gsub(/^\s*postgresql\s*([0-9.]+).*$/i, '\1')
+        if version >= '8.4'
+          modification_check = <<-end_sql
+            IF NEW IS NOT DISTINCT FROM OLD THEN
+              RETURN NULL;
+            END IF;
+          end_sql
+        else
+          modification_check = ""
+        end
+
         # now create the trigger
         execute(<<-end_sql)
           CREATE OR REPLACE FUNCTION "#{params[:trigger_name]}"() RETURNS TRIGGER AS $change_trigger$
@@ -61,6 +73,7 @@ module RR
                 INSERT INTO #{schema_prefix}#{params[:log_table]}(change_table, change_key, change_type, change_time)
                   SELECT '#{params[:table]}', #{key_clause('OLD', params)}, 'D', now();
               ELSIF (TG_OP = 'UPDATE') THEN
+                #{modification_check}
                 INSERT INTO  #{schema_prefix}#{params[:log_table]}(change_table, change_key, change_new_key, change_type, change_time)
                   SELECT '#{params[:table]}', #{key_clause('OLD', params)}, #{key_clause('NEW', params)}, 'U', now();
               ELSIF (TG_OP = 'INSERT') THEN

data/lib/rubyrep/replication_run.rb

@@ -11,6 +11,11 @@ module RR
     # The current TaskSweeper
     attr_accessor :sweeper
 
+    # An array of ReplicationDifference which originally failed replication but should be tried one more time
+    def second_chancers
+      @second_chancers ||= []
+    end
+
     # Returns the current ReplicationHelper; creates it if necessary
     def helper
       @helper ||= ReplicationHelper.new(self)
@@ -39,6 +44,20 @@ module RR
       end
     end
 
+    # Returns the next available ReplicationDifference.
+    # (Either new unprocessed differences or if not available, the first available 'second chancer'.)
+    #
+    def load_difference
+      @loaders ||= LoggedChangeLoaders.new(session)
+      @loaders.update # ensure the cache of change log records is up-to-date
+      diff = ReplicationDifference.new @loaders
+      diff.load
+      unless diff.loaded? or second_chancers.empty?
+        diff = second_chancers.shift
+      end
+      diff
+    end
+
     # Executes the replication run.
     def run
       return unless [:left, :right].any? do |database|
@@ -57,29 +76,36 @@ module RR
       # Check for this and if timed out, return (silently).
       return if sweeper.terminated?
 
-      loaders = LoggedChangeLoaders.new(session)
-
       success = false
       begin
         replicator # ensure that replicator is created and has chance to validate settings
 
         loop do
           begin
-            loaders.update # ensure the cache of change log records is up-to-date
-            diff = ReplicationDifference.new loaders
-            diff.load
+            diff = load_difference
             break unless diff.loaded?
             break if sweeper.terminated?
             if diff.type != :no_diff and not event_filtered?(diff)
               replicator.replicate_difference diff
             end
           rescue Exception => e
-            begin
-              helper.log_replication_outcome diff, e.message,
-                e.class.to_s + "\n" + e.backtrace.join("\n")
-            rescue Exception => _
-              # if logging to database itself fails, re-raise the original exception
-              raise e
+            if e.message =~ /violates foreign key constraint|foreign key constraint fails/i and !diff.second_chance?
+              # Note:
+              # Identifying the foreign key constraint violation via regular expression is
+              # database dependent and *dirty*.
+              # It would be better to use the ActiveRecord #translate_exception mechanism.
+              # However as per version 3.0.5 this doesn't work yet properly.
+
+              diff.second_chance = true
+              second_chancers << diff
+            else
+              begin
+                helper.log_replication_outcome diff, e.message,
+                  e.class.to_s + "\n" + e.backtrace.join("\n")
+              rescue Exception => _
+                # if logging to database itself fails, re-raise the original exception
+                raise e
+              end
             end
           end
         end

data/lib/rubyrep/replication_runner.rb

@@ -2,6 +2,12 @@ $LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
 
 require 'optparse'
 require 'thread'
+require 'monitor'
+
+class Monitor
+  alias lock mon_enter
+  alias unlock mon_exit
+end
 
 module RR
   # This class implements the functionality of the 'replicate' command.
@@ -94,7 +100,7 @@ EOS
     # Initializes the waiter thread used for replication pauses and processing
     # the process TERM signal.
     def init_waiter
-      @termination_mutex = Mutex.new
+      @termination_mutex = Monitor.new
       @termination_mutex.lock
       @waiter_thread ||= Thread.new {@termination_mutex.lock; self.termination_requested = true}
       %w(TERM INT).each do |signal|

data/lib/rubyrep/table_scan_helper.rb

@@ -19,7 +19,15 @@ module RR
       return 1 unless left_row 
       rank = 0
       primary_key_names.any? do |key|
-        rank = left_row[key] <=> right_row[key]
+        if left_row[key].kind_of?(String)
+          # When databases order strings, then 'a' < 'A' while for Ruby 'A' < 'a'
+          # ==> Use a combination of case sensitive and case insensitive comparing to
+          #     reproduce the database behaviour.
+          rank = left_row[key].casecmp(right_row[key]) # deal with 'a' to 'B' comparisons
+          rank = -(left_row[key] <=> right_row[key]) if rank == 0 # deal with 'a' to 'A' comparisons
+        else
+          rank = left_row[key] <=> right_row[key]
+        end
         rank != 0
       end
       rank

data/lib/rubyrep/version.rb

@@ -1,8 +1,8 @@
 module RR #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 1
-    MINOR = 1
-    TINY  = 2
+    MINOR = 2
+    TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/sims/performance/performance.rake

@@ -1,3 +1,4 @@
+
 require 'rake'
 require 'benchmark'
 
@@ -136,7 +137,7 @@ def populate_rep_data
     # Updating progress bar
     progress_bar.step
     
-    database = [:left, :right].rand
+    database = [:left, :right][rand(2)]
 
     case rand(100)
     when 0...BIG_REP_INSERT
@@ -147,7 +148,7 @@ def populate_rep_data
       next_id += 1
       session.send(database).insert_record 'big_rep', attributes
     when BIG_REP_INSERT...BIG_REP_UPDATE
-      id = all_ids[database].rand
+      id = all_ids[database][rand(all_ids[database].size)]
       attributes = session.send(database).select_one("select * from big_rep where id = '#{id}'")
       column = number_columns[rand(number_columns.size)]
       attributes[column] = rand(1000)

data/spec/connection_extenders_registration_spec.rb

@@ -10,9 +10,13 @@ describe ConnectionExtenders do
   it "db_connect should install the already created logger" do
     configuration = deep_copy(Initializer.configuration)
     io = StringIO.new
-    logger = Logger.new(io)
+    logger = ActiveSupport::BufferedLogger.new(io)
     configuration.left[:logger] = logger
     session = Session.new configuration
+
+    session.left.connection.instance_eval {@logger}.should == logger
+    session.right.connection.instance_eval {@logger}.should_not == logger
+
     session.left.select_one "select 'left_query'"
     session.right.select_one "select 'right_query'"
 

data/spec/proxy_connection_spec.rb

@@ -9,7 +9,7 @@ describe ProxyConnection do
   end
 
   it "initialize should connect to the database" do
-    @connection.connection.active?.should == true
+    (!!@connection.connection.active?).should == true
   end
 
   it "initialize should store the configuratin" do
@@ -17,9 +17,21 @@ describe ProxyConnection do
   end
   
   it "destroy should disconnect from the database" do
+    if ActiveSupport.const_defined?(:Notifications)
+      ConnectionExtenders::install_logger @connection.connection, :logger => StringIO.new
+      log_subscriber = @connection.connection.log_subscriber
+
+      ActiveSupport::Notifications.notifier.listeners_for("sql.active_record").should include(log_subscriber)
+    end
+
     @connection.destroy
 
-    @connection.connection.active?.should == false
+    if ActiveSupport.const_defined?(:Notifications)
+      ActiveSupport::Notifications.notifier.listeners_for("sql.active_record").should_not include(log_subscriber)
+      @connection.connection.log_subscriber.should be_nil
+    end
+
+    (!!@connection.connection.active?).should == false
   end
 
   it "cursors should return the current cursor hash or an empty hash if nil" do

data/spec/replication_run_spec.rb

@@ -135,6 +135,68 @@ describe ReplicationRun do
     end
   end
 
+  it "run should replication records with foreign key constraints" do
+    begin
+      config = deep_copy(standard_config)
+      config.options[:committer] = :never_commit
+
+      session = Session.new(config)
+
+      session.left.insert_record 'referencing_table', {
+        'id' => '5',
+      }
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'referencing_table',
+        'change_key' => 'id|5',
+        'change_type' => 'I',
+        'change_time' => Time.now
+      }
+
+      session.left.insert_record 'referenced_table2', {
+        'id' => '6',
+      }
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'referenced_table2',
+        'change_key' => 'id|6',
+        'change_type' => 'I',
+        'change_time' => Time.now
+      }
+
+      session.left.update_record 'referencing_table', {
+        'id' => 5,
+        'third_fk' => '6'
+      }
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'referencing_table',
+        'change_key' => 'id|5',
+        'change_new_key' => 'id|5',
+        'change_type' => 'U',
+        'change_time' => Time.now
+      }
+
+      run = ReplicationRun.new session, TaskSweeper.new(1)
+      run.run
+
+      session.right.select_record(:table => "referencing_table", :from => {'id' => 5}).should == {
+        'id' => 5,
+        'first_fk' => nil,
+        'second_fk' => nil,
+        'third_fk' => 6
+      }
+    ensure
+      Committers::NeverCommitter.rollback_current_session
+      if session
+        session.left.execute "delete from referencing_table where id = 5"
+        session.left.execute "delete from referenced_table2 where id = 6"
+
+        session.right.execute "delete from referencing_table where id = 5"
+        session.right.execute "delete from referenced_table2 where id = 6"
+        
+        session.left.execute "delete from rr_pending_changes"
+      end
+    end
+  end
+
   it "run should not replicate filtered changes" do
     begin
       config = deep_copy(standard_config)

data/spec/table_scan_helper_spec.rb

@@ -13,6 +13,13 @@ describe TableScanHelper do
     @scan.rank_rows({'first_id' => 1, 'second_id' => 1}, {'first_id' => 1, 'second_id' => 1}).should == 0
     @scan.rank_rows({'first_id' => 1, 'second_id' => 1}, {'first_id' => 1, 'second_id' => 2}).should == -1
     @scan.rank_rows({'first_id' => 2, 'second_id' => 1}, {'first_id' => 1, 'second_id' => 1}).should == 1
+
+    # should rank strings according to database logic ('a' < 'A')
+    # instead of the Ruby logic (which is the other way round)
+    @scan.rank_rows({'first_id' => 'a', 'second_id' => 1}, {'first_id' => 'B', 'second_id' => 1}).should == -1
+    @scan.rank_rows({'first_id' => 'a', 'second_id' => 1}, {'first_id' => 'A', 'second_id' => 1}).should == -1
+    @scan.rank_rows({'first_id' => 'a', 'second_id' => 1}, {'first_id' => 'a', 'second_id' => 1}).should == 0
+
     lambda {@scan.rank_rows(nil,nil)}.should raise_error(RuntimeError, 'At least one of left_row and right_row must not be nil!')
     @scan.rank_rows(nil, {'first_id' => 1, 'second_id' => 1}).should == 1
     @scan.rank_rows({'first_id' => 1, 'second_id' => 1}, nil).should == -1

data/tasks/database.rake

@@ -18,7 +18,7 @@ def create_database(config)
       @charset   = ENV['CHARSET']   || 'utf8'
       @collation = ENV['COLLATION'] || 'utf8_general_ci'
       begin
-        connection = RR::ConnectionExtenders.db_connect(config.merge({'database' => nil}))
+        connection = RR::ConnectionExtenders.db_connect(config.merge({:database => nil}))
         connection.create_database(config[:database], {:charset => @charset, :collation => @collation})
         RR::ConnectionExtenders.db_connect(config)
       rescue

data/tasks/rspec.rake

@@ -41,18 +41,14 @@ namespace :spec do
   desc "Run the specs for all supported databases"
   task :all_dbs do
     [:postgres, :mysql].each do |test_db|
-      puts "Running specs for #{test_db.id2name}"
-      ENV['RR_TEST_DB'] = test_db.id2name
-      system "spec spec"
+      puts "Running specs for #{test_db}"
+      system "bash -c 'RR_TEST_DB=#{test_db} spec spec'"
     end
   end
   
   desc "Run the specs for all supported databases and ruby platforms" 
   task :all_rubies do
-    puts "Running spec:all_dbs in standard ruby"
-    system "rake spec:all_dbs"
-    puts "Running spec:all_dbs in jruby"
-    system "export PATH=#{JRUBY_HOME}/bin:$PATH; rake spec:all_dbs"
+    system %(rvm exec bash -c 'for db in postgres mysql; do echo "`rvm current` - $db:"; RR_TEST_DB=$db spec spec; done')
   end
   
   begin

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: rubyrep
 version: !ruby/object:Gem::Version 
-  prerelease: false
+  hash: 31
+  prerelease: 
   segments: 
   - 1
-  - 1
   - 2
-  version: 1.1.2
+  - 0
+  version: 1.2.0
 platform: ruby
 authors: 
 - Arndt Lehmann
@@ -14,53 +15,60 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-10 00:00:00 +09:00
+date: 2011-03-07 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: activesupport
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 13
         segments: 
-        - 2
         - 3
+        - 0
         - 5
-        version: 2.3.5
+        version: 3.0.5
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: activerecord
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 13
         segments: 
-        - 2
         - 3
+        - 0
         - 5
-        version: 2.3.5
+        version: 3.0.5
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: hoe
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 41
         segments: 
         - 2
-        - 4
-        - 0
-        version: 2.4.0
+        - 9
+        - 1
+        version: 2.9.1
   type: :development
   version_requirements: *id003
-description: Asynchronous master-master replication of relational databases.
-email: mail@arndtlehman.com
+description: ""
+email: 
+- mail@arndtlehman.com
 executables: 
 - rubyrep
 extensions: []
@@ -222,8 +230,9 @@ files:
 - tasks/stats.rake
 - tasks/task_helper.rb
 - tasks/website.rake
+- .gemtest
 has_rdoc: true
-homepage: http://rubyrep.rubyforge.org
+homepage: 
 licenses: []
 
 post_install_message: 
@@ -233,25 +242,29 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: rubyrep
-rubygems_version: 1.3.6
+rubygems_version: 1.5.2
 signing_key: 
 specification_version: 3
-summary: Asynchronous master-master replication of relational databases.
+summary: ""
 test_files: []