rubyrep 1.2.0 → 2.0.0

data/lib/rubyrep/proxy_connection.rb

@@ -66,9 +66,27 @@ module RR
         end
 
         self.rows = connection.select_all query
+
+        ###############
+        # Below approach can only be used starting with activerecord version 5.
+        # And as of 2017-05-18 jruby (i.e. version 9.1.7.0) supports only up to activerecord version 4.2.
+        ###############
+        # result = connection.select_all(query)
+        # column_types = result.column_types
+        # columns = result.columns
+        # identity_type = ActiveRecord::Type::Value.new
+        # types = columns.map { |name| column_types.fetch(name, identity_type) }
+        # self.rows = result.rows.map do |values|
+        #   row = {}
+        #   columns.zip(types, values).each do |name, type, value|
+        #     row[name] = type.deserialize(value)
+        #   end
+        #   row
+        # end
+
         self.current_row_index = 0
       end
-      self.current_row_index < self.rows.size
+      self.current_row_index < self.rows.length
     end
 
     # Returns the row as a column => value hash and moves the cursor to the next row.
@@ -77,7 +95,7 @@ module RR
       self.last_row = self.rows[self.current_row_index]
       self.current_row_index += 1
 
-      if self.current_row_index == self.rows.size
+      if self.current_row_index == self.rows.length
         self.rows = nil
       end
 
@@ -184,20 +202,12 @@ module RR
     # * first build the query based on options forwarded to #table_select_query
     # +options+ is a hash with
     # * :+query+: executes the given query
-    # * :+type_cast+:
-    #   Unless explicitely disabled with +false+, build type casting cursor
-    #   around result.
-    # * :+table+:
-    #   Name of the table from which to read data.
-    #   Required unless type casting is disabled.
     # * further options as taken by #table_select_query to build the query
     # * :+row_buffer_size+:
     #   Integer controlling how many rows a read into memory at one time.
     def select_cursor(options)
       cursor = ResultFetcher.new(self, options)
-      unless options[:type_cast] == false
-        cursor = TypeCastingCursor.new(self, options[:table], cursor)
-      end
+      cursor = TypeCastingCursor.new(self, options[:table], cursor)
       cursor
     end
 
@@ -255,7 +265,7 @@ module RR
         table_columns[table] = {}
         columns(table).each {|c| table_columns[table][c.name] = c}
       end
-      connection.quote value, table_columns[table][column]
+      connection.column_aware_quote value, table_columns[table][column.to_s]
     end
     
     # Create a cursor for the given table.

data/lib/rubyrep/replication_difference.rb

@@ -93,7 +93,7 @@ module RR
 
     # Prevents session and change loaders from going into YAML output
     def to_yaml_properties
-      instance_variables.sort.reject {|var_name| ['@session', '@loaders'].include? var_name}
+      instance_variables.sort.reject {|var_name| [:'@session', :'@loaders'].include? var_name}
     end
 
   end

data/lib/rubyrep/replication_extenders/mysql_replication.rb

@@ -3,7 +3,7 @@ module RR
 
     # Provides Mysql specific functionality for database replication
     module MysqlReplication
-      RR::ReplicationExtenders.register :mysql => self
+      RR::ReplicationExtenders.register :mysql2 => self
 
       # Creates or replaces the replication trigger function.
       # See #create_replication_trigger for a descriptions of the +params+ hash.

data/lib/rubyrep/replication_helper.rb

@@ -92,7 +92,7 @@ module RR
       columns = @table_columns[table]
       type_casted_row = {}
       row.each_pair do |column_name, value|
-        type_casted_row[column_name] = columns[column_name].type_cast(value)
+        type_casted_row[column_name] = session.left.connection.fixed_type_cast value, columns[column_name]
       end
       type_casted_row
     end

data/lib/rubyrep/replication_runner.rb

@@ -128,10 +128,19 @@ EOS
       raise e
     end
 
+    # For testing: will be called after replication preparation is finished
+    def replication_preparation_finished
+    end
+
+    # For testing: will be called after a replication run is finished
+    def replication_run_finished
+    end
+
     # Executes an endless loop of replication runs
     def execute
       init_waiter
       prepare_replication
+      replication_preparation_finished
 
       until termination_requested do
         begin
@@ -145,6 +154,7 @@ EOS
           end
         end
         pause_replication
+        replication_run_finished
       end
     end
 

data/lib/rubyrep/scan_report_printers/scan_detail_reporter.rb

@@ -16,7 +16,7 @@ module RR::ScanReportPrinters
       "The 'mode' argument determines how the row differences are printed:",
       " * full shows the full records",
       " * keys shows the primary key columns only",
-      " * diff shows the primary key and differing columsn only"
+      " * diff shows the primary key and differing columns only"
     
     # The current Session object
     attr_accessor :session

data/lib/rubyrep/table_spec_resolver.rb

@@ -88,7 +88,7 @@ module RR
         case table_spec
         when /^\/.*\/$/ # matches e. g. '/^user/'
           table_spec = table_spec.sub(/^\/(.*)\/$/,'\1') # remove leading and trailing slash
-          matching_tables = tables(:left).grep(Regexp.new(table_spec, Regexp::IGNORECASE, 'U'))
+          matching_tables = tables(:left).grep(Regexp.new(table_spec, Regexp::IGNORECASE))
           matching_tables.each do |table|
             if !verify or tables(:right).include? table
               table_pairs << {:left => table, :right => table}

data/lib/rubyrep/type_casting_cursor.rb

@@ -6,13 +6,15 @@ module RR
     # Delegate the uninteresting methods to the original cursor
     def next?; org_cursor.next? end
     def clear; org_cursor.clear end
-    
-    # The original cursor object
+    def connection; org_cursor.connection end
+    def options; org_cursor.options end
+
+    # @return [ResultFetcher] the original cursor
     attr_accessor :org_cursor
     
     # A column_name => Column cache
     attr_accessor :columns
-    
+
     # Creates a new TypeCastingCursor based on provided database connection and table name
     # for the provided database query cursor
     def initialize(connection, table, cursor)
@@ -24,7 +26,9 @@ module RR
     # Reads the next row from the original cursor and returns the row with the type casted row values.
     def next_row
       row = org_cursor.next_row
-      row.each {|column, value| row[column] = columns[column].type_cast value}
+      row.each do |column, value|
+        row[column] = connection.connection.fixed_type_cast value, columns[column]
+      end
       row
     end    
   end

data/lib/rubyrep/version.rb

@@ -1,9 +1,3 @@
 module RR #:nodoc:
-  module VERSION #:nodoc:
-    MAJOR = 1
-    MINOR = 2
-    TINY  = 0
-
-    STRING = [MAJOR, MINOR, TINY].join('.')
-  end
+  VERSION = "2.0.0"
 end

data/lib/rubyrep.rb

@@ -4,8 +4,10 @@ $LOAD_PATH.unshift File.dirname(__FILE__) + "/rubyrep"
 require 'rubygems'
 require 'yaml'
 
-gem 'activerecord', '>= 3.0.5'
+gem 'activerecord', '>= 4.2', '< 5'
 require 'active_record'
+require 'activerecord-jdbc-adapter' if RUBY_PLATFORM == 'java'
+require 'ap'
 
 require 'version'
 require 'configuration'
@@ -56,8 +58,7 @@ require 'generate_runner'
 require 'noisy_connection'
 
 Dir["#{File.dirname(__FILE__)}/rubyrep/connection_extenders/*.rb"].each do |extender|
-  # jdbc_extender.rb is only loaded if we are running on jruby
-  require extender unless extender =~ /jdbc/ and not RUBY_PLATFORM =~ /java/
+  require extender
 end
 
 require 'replication_initializer'

data/rubyrep

@@ -5,4 +5,8 @@ script_dir="`dirname \"$0\"`"
 jruby_path="$script_dir"/jruby/bin/jruby
 rubyrep_path="$script_dir"/bin/rubyrep
 
+# ensure isolation from rvm or other possible existing ruby installations
+unset GEM_HOME
+unset GEM_PATH
+
 $jruby_path --server $rubyrep_path $*

data/rubyrep.bat

@@ -1,4 +1,9 @@
 @echo off
+
+REM ensure isolation from rvm or other possibly existing ruby installations
+set GEM_HOME=
+set GEM_PATH=
+
 set jruby_path=%~dp0jruby\bin\jruby.bat
 set rubyrep_path=%~dp0bin\rubyrep
 %jruby_path% --server %rubyrep_path% %1 %2 %3 %4 %5 %6 %7 %8 %9

data/rubyrep.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rubyrep/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "rubyrep"
+  spec.version       = RR::VERSION
+  spec.authors       = ["Arndt Lehmann"]
+  spec.email         = ["arndtlehmann@arndtlehmann.com"]
+
+  spec.summary       = %q{Open-source solution for asynchronous, master-master replication of relational databases.}
+  spec.homepage      = "http://www.rubyrep.org"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "bin"
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "rspec", "~> 3.5"
+
+  spec.add_runtime_dependency "activerecord", "~> 4.2"
+  spec.add_runtime_dependency "awesome_print"
+end

data/sims/performance/big_rep_spec.rb

@@ -34,6 +34,18 @@ describe "Big Rep" do
       record_quantity(session, :right, 'rr_pending_changes')
   end
 
+  # Removes test data
+  #
+  # @param [Session] session the target session
+  # @param [Initializer] initializer the target initializer
+  def cleanup(session, initializer)
+    [:left, :right].each do |database|
+      initializer.drop_trigger database, 'big_rep' if initializer.trigger_exists? database, 'big_rep'
+      session.send(database).execute "delete from big_rep"
+      session.send(database).execute "delete from rr_pending_changes"
+    end
+  end
+
   # Runs a replication of the big_rep table.
   def run_rep
     config = deep_copy(Initializer.configuration)
@@ -41,32 +53,41 @@ describe "Big Rep" do
       :committer => :buffered_commit,
       :replication_conflict_handling => :later_wins,
     }
+    config.options[:event_filter] = filter = {}
 
     session = Session.new config
     initializer = ReplicationInitializer.new session
     begin
+      cleanup session, initializer
       [:left, :right].each do |database|
         session.send(database).execute "insert into big_rep select * from big_rep_backup"
         session.send(database).execute "insert into rr_pending_changes select * from big_rep_pending_changes"
         initializer.create_trigger database, 'big_rep'
       end
 
-      puts "\nReplicating (#{session.proxied? ? :proxied : :direct}) table big_rep (#{number_changes(session)} changes)"
-
-      t = Thread.new do
-        remaining_changes = number_changes(session)
-        progress_bar = ScanProgressPrinters::ProgressBar.new(remaining_changes, session, 'big_rep', 'big_rep')
-        while remaining_changes > 0
-          sleep 1
-          new_remaining_changes = number_changes(session)
-          progress_bar.step remaining_changes - new_remaining_changes
-          remaining_changes = new_remaining_changes
+      filter[:number_changes] = lambda { number_changes(session)}
+      def filter.before_replicate(*args)
+        if self[:last_check].nil? || self[:last_check] < 1.second.ago || args.empty?
+          self[:last_check] = Time.now
+          new_remaining_changes = self[:number_changes].call
+          steps = self[:remaining_changes] - new_remaining_changes
+          self[:progress_bar].step steps
+          self[:remaining_changes] = new_remaining_changes
         end
+        true
       end
+      filter[:remaining_changes] = filter[:number_changes].call
+      filter[:progress_bar] = ScanProgressPrinters::ProgressBar.new(
+        filter[:remaining_changes],
+        session,
+        'big_rep',
+        'big_re'
+      )
+
+      puts "\nReplicating (#{session.proxied? ? :proxied : :direct}) table big_rep (#{number_changes(session)} changes)"
 
       run = ReplicationRun.new session, TaskSweeper.new(5)
-      benchmark = Benchmark.measure { run.run }
-      t.join 10
+      benchmark = Benchmark.measure { run.run; filter.before_replicate }
       puts "\n  time required: #{benchmark}"
 
       left_fingerprint = {
@@ -79,11 +100,7 @@ describe "Big Rep" do
       }
       left_fingerprint.should == right_fingerprint
     ensure
-      [:left, :right].each do |database|
-        initializer.drop_trigger database, 'big_rep'
-        session.send(database).execute "delete from big_rep"
-        session.send(database).execute "delete from rr_pending_changes"
-      end
+      cleanup session, initializer
     end
   end
 

data/sims/performance/performance.rake

@@ -2,7 +2,7 @@
 require 'rake'
 require 'benchmark'
 
-require File.dirname(__FILE__) + '/../sim_helper'
+require File.expand_path(File.dirname(__FILE__) + '/../sim_helper')
 
 # Prepares the database schema for the performance tests.
 def prepare_schema
@@ -32,20 +32,20 @@ BIG_SCAN_MODIFIED = BIG_SCAN_SAME + 3
 BIG_SCAN_LEFT_ONLY = BIG_SCAN_MODIFIED + 1 # difference to 100% will be right_only records
 
 def big_scan_columns
-  @@big_scan_columns ||= nil
-  unless @@big_scan_columns
+  @big_scan_columns ||= nil
+  unless @big_scan_columns
     session = RR::Session.new
-    @@big_scan_columns = session.left.column_names('big_scan')
+    @big_scan_columns = session.left.column_names('big_scan')
   end
-  @@big_scan_columns
+  @big_scan_columns
 end
 
 def text_columns
-  @@text_columns ||= big_scan_columns.select {|column_name| column_name =~ /^text/}
+  @text_columns ||= big_scan_columns.select {|column_name| column_name =~ /^text/}
 end
 
 def number_columns
-  @@number_columns ||= big_scan_columns.select {|column_name| column_name =~ /^number/}
+  @number_columns ||= big_scan_columns.select {|column_name| column_name =~ /^number/}
 end
 
 def random_attributes
@@ -193,37 +193,17 @@ namespace :sims do
     
     desc "Runs the big_scan simulation"
     task :scan do
-      Spec::Runner::CommandLine.run(
-        Spec::Runner::OptionParser.parse(
-          ['--options', "spec/spec.opts", "./sims/performance/big_scan_spec.rb"],
-          $stdout, $stderr))
+      system "rspec sims/performance/big_scan_spec.rb"
     end
     
     desc "Runs the big_sync simulation"
     task :sync do
-      Spec::Runner::CommandLine.run(
-        Spec::Runner::OptionParser.parse(
-          ['--options', "spec/spec.opts", "./sims/performance/big_sync_spec.rb"],
-          $stdout, $stderr))
+      system "rspec sims/performance/big_sync_spec.rb"
     end
 
     desc "Runs the big_rep simulation"
     task :rep do
-      Spec::Runner::CommandLine.run(
-        Spec::Runner::OptionParser.parse(
-          ['--options', "spec/spec.opts", "./sims/performance/big_rep_spec.rb"],
-          $stdout, $stderr))
-    end
-
-    begin
-      require 'ruby-prof/task'
-      RubyProf::ProfileTask.new do |t|
-        t.test_files = FileList["./sims/performance/*_spec.rb"]
-        t.output_dir = 'profile'
-        t.printer = :flat
-        t.min_percent = 1
-      end
-    rescue LoadError
+      system "rspec sims/performance/big_rep_spec.rb"
     end
   end
-end
+end

data/tasks/database.rake

@@ -2,19 +2,19 @@ $LOAD_PATH.unshift File.dirname(__FILE__) + "../lib/rubyrep"
 require 'rake'
 require 'rubyrep'
 
-require File.dirname(__FILE__) + '/task_helper.rb'
-require File.dirname(__FILE__) + '/../config/test_config'
-require File.dirname(__FILE__) + '/../spec/spec_helper'
+require_relative 'task_helper.rb'
+require_relative '../config/test_config'
+require_relative '../spec/spec_helper'
 
 # Creates the databases for the given configuration hash
 def create_database(config)
   begin
     RR::ConnectionExtenders.db_connect(config)
-  rescue
+  rescue StandardError, LoadError
     case config[:adapter]
     when 'postgresql'
       `createdb "#{config[:database]}" -E utf8`
-    when 'mysql'
+    when 'mysql2'
       @charset   = ENV['CHARSET']   || 'utf8'
       @collation = ENV['COLLATION'] || 'utf8_general_ci'
       begin
@@ -62,11 +62,11 @@ def create_postgres_schema(config)
     end
     execute "insert into rr_simple(id, name) values(1, 'bla')"
 
-    create_table :rr_referenced, :id => true do |t|
+    create_table :rr_referenced do |t|
       t.column :name, :string
     end
 
-    create_table :rr_referencing, :id => true do |t|
+    create_table :rr_referencing do |t|
       t.column :rr_referenced_id, :integer
     end
 
@@ -166,11 +166,11 @@ def create_sample_schema(database, config)
         PRIMARY KEY (first_id, second_id)
     end_sql
 
-    create_table :referenced_table2, :id => true do |t|
+    create_table :referenced_table2 do |t|
       t.column :name, :string
     end
 
-    create_table :referencing_table, :id => true do |t|
+    create_table :referencing_table do |t|
       t.column :first_fk, :integer
       t.column :second_fk, :integer
       t.column :third_fk, :integer
@@ -367,7 +367,7 @@ def drop_sample_schema(config)
 end
 
 class ExtenderCombinedKey < ActiveRecord::Base
-  set_table_name "extender_combined_key"
+  self.table_name = "extender_combined_key"
   include CreateWithKey  
 end
 
@@ -376,17 +376,17 @@ class ScannerRecords < ActiveRecord::Base
 end
 
 class ScannerLeftRecordsOnly < ActiveRecord::Base
-  set_table_name "scanner_left_records_only"
+  self.table_name = "scanner_left_records_only"
   include CreateWithKey
 end
 
 class ExtenderOneRecord < ActiveRecord::Base
-  set_table_name "extender_one_record"
+  self.table_name = "extender_one_record"
   include CreateWithKey
 end
 
 class ExtenderTypeCheck < ActiveRecord::Base
-  set_table_name "extender_type_check"
+  self.table_name = "extender_type_check"
   include CreateWithKey
 end
 
@@ -516,4 +516,4 @@ namespace :db do
       end
     end
   end
-end
+end

data/tasks/java.rake

@@ -2,15 +2,28 @@ namespace :deploy do
 
   desc "Create the java installation package"
   task :java do
-    pkg_name = "rubyrep-#{RR::VERSION::STRING}"
+    jruby_version='9.1.10.0'
+    pkg_name = "rubyrep-#{RR::VERSION}"
 
     system "rm -rf /tmp/#{pkg_name}"
     system "mkdir /tmp/#{pkg_name}"
     system "git archive master |tar -x -C /tmp/#{pkg_name}"
-    system "mkdir -p /tmp/#{pkg_name}/jruby"
-    system "cp -r #{JRUBY_HOME}/* /tmp/#{pkg_name}/jruby/"
-    system "cd /tmp/#{pkg_name}/jruby; rm -rf samples share/ri lib/ruby/gems/1.8/doc"
-    system "chmod a+x /tmp/#{pkg_name}/rubyrep"
+    system "curl -o /tmp/#{pkg_name}/jruby.tar.gz https://s3.amazonaws.com/jruby.org/downloads/#{jruby_version}/jruby-bin-#{jruby_version}.tar.gz"
+    system "tar -C /tmp/#{pkg_name} -xzf /tmp/#{pkg_name}/jruby.tar.gz"
+    system "mv /tmp/#{pkg_name}/jruby-#{jruby_version} /tmp/#{pkg_name}/jruby"
+    system "rm /tmp/#{pkg_name}/jruby.tar.gz"
+    system %[
+      cd /tmp/#{pkg_name}
+      export PATH=`pwd`/jruby/bin:$PATH
+      unset GEM_HOME
+      unset GEM_PATH
+      gem install activerecord -v 4.2.8
+      gem install jdbc-mysql -v 5.1.42
+      gem install jdbc-postgres -v 9.4.1206
+      gem install activerecord-jdbcmysql-adapter -v 1.3.23
+      gem install activerecord-jdbcpostgresql-adapter -v 1.3.23
+      gem install awesome_print -v 1.7.0
+    ]
     system "cd /tmp; rm -f #{pkg_name}.zip; zip -r #{pkg_name}.zip #{pkg_name} >/dev/null"
     system "mkdir -p pkg"
     system "cp /tmp/#{pkg_name}.zip pkg"

data/tasks/rspec.rake

@@ -1,11 +1,11 @@
 begin
-  require 'spec'
+  require 'rspec'
 rescue LoadError
   require 'rubygems'
-  require 'spec'
+  require 'rspec'
 end
 begin
-  require 'spec/rake/spectask'
+require 'rspec/core/rake_task'
 rescue LoadError
   puts <<-EOS
 To use rspec for testing you must install rspec gem:
@@ -14,51 +14,31 @@ To use rspec for testing you must install rspec gem:
   exit(0)
 end
 
-desc "Run the specs under spec/models"
-Spec::Rake::SpecTask.new do |t|
-  t.spec_opts = ['--options', "spec/spec.opts"]
-  t.spec_files = FileList['spec/*_spec.rb']
+task :spec do
+  system "rspec spec"
 end
 
 namespace :spec do
   desc "Generate specdocs for examples for inclusion in RDoc"
-  Spec::Rake::SpecTask.new('docs') do |t|
-    t.spec_files = FileList['spec/*_spec.rb']
-    t.spec_opts = ["--format", "specdoc"]
+  task :docs do
+    system 'rspec --format documentation spec'
   end
 
-  desc "Run the specs with RCov"
-  Spec::Rake::SpecTask.new('rcov') do |t|
-    t.spec_opts = ['--options', "spec/spec.opts"]
-    t.spec_files = FileList['spec/*_spec.rb']
-    t.rcov = true
-    t.rcov_opts = [
-      '--exclude', 'tasks/,spec/,gems/\(?!rubyrep\)',
-      '--xrefs'
-    ]
+  desc "Run the specs with coverage"
+  task :cov do
+    system 'COVERAGE=true rspec spec'
   end
-  
+
   desc "Run the specs for all supported databases"
   task :all_dbs do
     [:postgres, :mysql].each do |test_db|
       puts "Running specs for #{test_db}"
-      system "bash -c 'RR_TEST_DB=#{test_db} spec spec'"
+      system "bash -c 'RR_TEST_DB=#{test_db} rspec spec'"
     end
   end
   
   desc "Run the specs for all supported databases and ruby platforms" 
   task :all_rubies do
-    system %(rvm exec bash -c 'for db in postgres mysql; do echo "`rvm current` - $db:"; RR_TEST_DB=$db spec spec; done')
-  end
-  
-  begin
-    require 'ruby-prof/task'
-    RubyProf::ProfileTask.new do |t|
-      t.test_files = FileList['spec/*_spec.rb']
-      t.output_dir = 'profile'
-      t.printer = :flat
-      t.min_percent = 1
-    end
-  rescue LoadError
+    system %(rvm ruby@rubyrep,jruby@rubyrep do bash -c 'for db in postgres mysql; do echo "`rvm current` - $db:"; RR_TEST_DB=$db rspec spec; done')
   end
-end
+end

data/tasks/stats.rake

@@ -1,19 +1,4 @@
 begin
-  require 'code_statistics' # This library is coming with the rails gem
-  desc "Report code statistics (KLOCs, etc)"
-  task :stats do
-    STATS_DIRECTORIES = [
-      %w(Libraries          lib/),
-      %w(Unit\ tests        spec/),
-      %w(Integration\ tests sims/)
-    ].collect { |name, dir| [ name, "#{File.dirname(__FILE__)}/../#{dir}" ] }.select { |name, dir| File.directory?(dir) }
-
-    desc "Report code statistics (KLOCs, etc) from the application"
-    task :stats do
-      require 'code_statistics'
-      CodeStatistics.new(*STATS_DIRECTORIES).to_s
-    end
-    
-  end
+  require 'code_statistics' # install this gem to get a task 'stats'
 rescue LoadError
 end