rubyrep 1.0.0

data/sims/performance/big_rep_spec.rb

@@ -0,0 +1,100 @@
+require 'benchmark'
+
+require File.dirname(__FILE__) + '/../../spec/spec_helper.rb'
+require File.dirname(__FILE__) + '/../sim_helper'
+
+include RR
+
+describe "Big Rep" do
+  before(:each) do
+  end
+
+  # Calculates and returns the number of records of the specified table.
+  # * +session+ current Session instance
+  # * +database+: designates database, either :left or :right
+  # * +table+: name of the table
+  def record_quantity(session, database, table = 'big_rep')
+    session.send(database).select_one( \
+        "select count(id) as count from #{table}")['count'].to_i
+  end
+
+  # Calculates and returns the sum of the number fields of records.
+  # * +session+: current Session instance
+  # * +database+: designates database, either :left or :right
+  def record_sum(session, database)
+    session.send(database).select_one( \
+        "select sum(number1) + sum(number2) + sum(number3) + sum(number4) as sum
+         from big_rep")['sum'].to_f
+  end
+
+  # Returns the number of changes that remain to be replicated
+  # * +session+: current Session instance
+  def number_changes(session)
+    record_quantity(session, :left, 'rr_pending_changes') +
+      record_quantity(session, :right, 'rr_pending_changes')
+  end
+
+  # Runs a replication of the big_rep table.
+  def run_rep
+    config = deep_copy(Initializer.configuration)
+    config.options = {
+      :committer => :buffered_commit,
+      :replication_conflict_handling => :later_wins,
+    }
+
+    session = Session.new config
+    initializer = ReplicationInitializer.new session
+    begin
+      [:left, :right].each do |database|
+        session.send(database).execute "insert into big_rep select * from big_rep_backup"
+        session.send(database).execute "insert into rr_pending_changes select * from big_rep_pending_changes"
+        initializer.create_trigger database, 'big_rep'
+      end
+
+      puts "\nReplicating (#{session.proxied? ? :proxied : :direct}) table big_rep (#{number_changes(session)} changes)"
+
+      t = Thread.new do
+        remaining_changes = number_changes(session)
+        progress_bar = ScanProgressPrinters::ProgressBar.new(remaining_changes, session, 'big_rep', 'big_rep')
+        while remaining_changes > 0
+          sleep 1
+          new_remaining_changes = number_changes(session)
+          progress_bar.step remaining_changes - new_remaining_changes
+          remaining_changes = new_remaining_changes
+        end
+      end
+
+      run = ReplicationRun.new session
+      benchmark = Benchmark.measure { run.run }
+      t.join 10
+      puts "\n  time required: #{benchmark}"
+
+      left_fingerprint = {
+        :quantity => record_quantity(session, :left),
+        :sum => record_sum(session, :left)
+      }
+      right_fingerprint = {
+        :quantity => record_quantity(session, :right),
+        :sum => record_sum(session, :right)
+      }
+      left_fingerprint.should == right_fingerprint
+    ensure
+      [:left, :right].each do |database|
+        initializer.drop_trigger database, 'big_rep'
+        session.send(database).execute "delete from big_rep"
+        session.send(database).execute "delete from rr_pending_changes"
+      end
+    end
+  end
+
+  it "Direct Replication should replicate correctly" do
+    Initializer.configuration = standard_config
+    run_rep
+  end
+
+  it "Proxied Replication should replicate correctly" do
+    ensure_proxy
+    Initializer.configuration = proxied_config
+    run_rep
+  end
+end

data/sims/performance/big_scan_spec.rb

@@ -0,0 +1,57 @@
+require 'benchmark'
+
+require File.dirname(__FILE__) + '/../../spec/spec_helper.rb'
+require File.dirname(__FILE__) + '/../sim_helper'
+
+include RR
+
+describe "Big Scan" do
+  before(:each) do
+  end
+
+  # Runs a scan of the big_scan table
+  def run_scan
+    session = Session.new
+    expected_result = {}
+    expected_result[:conflict] = session.left.select_one( \
+        "select count(id) as count from big_scan where diff_type = 'conflict'")['count'].to_i
+    expected_result[:left] = session.left.select_one( \
+        "select count(id) as count from big_scan where diff_type = 'left'")['count'].to_i
+    expected_result[:right] = session.right.select_one( \
+        "select count(id) as count from big_scan where diff_type = 'right'")['count'].to_i
+    
+    number_records = session.left.select_one( \
+        "select count(id) as count from big_scan")['count'].to_i \
+      + expected_result[:right]
+    number_differences = expected_result.values.inject {|sum, n| sum + n }
+    received_result = {:conflict => 0, :left => 0, :right => 0}
+
+    table_scan_class = TableScanHelper.scan_class(session)
+    puts "\nScanning table big_scan (#{number_differences} differences in #{number_records} records) using #{table_scan_class.name}"
+
+    scan = table_scan_class.new session, 'big_scan'
+    scan.progress_printer = RR::ScanProgressPrinters::ProgressBar
+    benchmark = Benchmark.measure {
+      scan.run do |diff_type, row|
+        received_result[diff_type] += 1
+      end
+    }
+    puts "\n  time required: #{benchmark}"
+    
+    received_result.should == expected_result
+  end
+  
+  it "ProxiedTableScan should identify differences between big_scan tables correctly" do
+    Initializer.configuration = deep_copy(proxied_config)
+    Initializer.configuration.options[:proxy_block_size] = 100
+    ensure_proxy
+    
+    run_scan
+  end
+
+  it "DirectTableScan should identify differences between big_scan tables correctly" do
+    Initializer.configuration = standard_config
+    
+    run_scan
+  end
+end

data/sims/performance/big_sync_spec.rb

@@ -0,0 +1,141 @@
+require 'benchmark'
+
+require File.dirname(__FILE__) + '/../../spec/spec_helper.rb'
+require File.dirname(__FILE__) + '/../sim_helper'
+
+include RR
+
+describe "Big Sync" do
+  before(:each) do
+  end
+
+  # Calculates and returns number of records of a given type.
+  # * :session: current Session instance
+  # * :record_type: type of records (String)
+  # * :database: designates database, either :left or :right
+  def record_quantity(session, record_type, database)
+    session.send(database).select_one( \
+        "select count(id) as count from big_scan where diff_type = '#{record_type}'")['count'].to_i
+  end
+
+  # Calculates and returns the sum of the number fields of records of the given type.
+  # * :session: current Session instance
+  # * :record_type: type of records (:left, :right, :same or :conflict)
+  # * :database: designates database, either :left or :right
+  def record_sum(session, record_type, database)
+    session.send(database).select_one( \
+        "select sum(number1) + sum(number2) + sum(number3) + sum(number4) as sum
+         from big_scan where diff_type = '#{record_type}'")['sum'].to_f
+  end
+
+  # Runs a sync of the big_scan table.
+  # * session: the current Session
+  # * expected_result: a hash of record quantities that should result.
+  def run_sync(session, expected_result)
+    begin
+      number_records =
+        record_quantity(session, :left, :left) +
+        record_quantity(session, :conflict, :left) +
+        record_quantity(session, :same, :left) +
+        record_quantity(session, :right, :right)
+
+      number_differences =
+        record_quantity(session, :left, :left) +
+        record_quantity(session, :conflict, :left) +
+        record_quantity(session, :right, :right)
+
+      puts "\nSyncing (#{Initializer.configuration.options[:syncer]}, #{session.proxied? ? :proxied : :direct}) table big_scan (#{number_differences} differences in #{number_records} records)"
+
+      sync = TableSync.new(session, 'big_scan')
+      sync.progress_printer = RR::ScanProgressPrinters::ProgressBar
+      benchmark = Benchmark.measure { sync.run { |diff_type, row| } }
+      puts "\n  time required: #{benchmark}"
+
+      {
+        :conflict_on_left => record_quantity(session, :conflict, :left),
+        :conflict_on_right => record_quantity(session, :conflict, :right),
+        :conflict_sum_on_left => record_sum(session, :conflict, :left),
+        :conflict_sum_on_right => record_sum(session, :conflict, :right),
+        :left_on_left => record_quantity(session, :left, :left),
+        :left_on_right => record_quantity(session, :left, :right),
+        :right_on_left => record_quantity(session, :right, :left),
+        :right_on_right => record_quantity(session, :right, :right)
+      }.should == expected_result
+    ensure
+      Committers::NeverCommitter.rollback_current_session
+    end
+  end
+
+  it "Proxied OneWaySync should sync correctly" do
+    ensure_proxy
+    Initializer.configuration = deep_copy(proxied_config)
+    
+    Initializer.configuration.options = {
+      :committer => :never_commit,
+      :syncer => :one_way,
+      :direction => :right,
+      :delete => true,
+      :proxy_block_size => 100,
+    }
+    
+    session = Session.new
+    expected_result = {
+      :conflict_on_left => record_quantity(session, :conflict, :left),
+      :conflict_on_right => record_quantity(session, :conflict, :right),
+      :conflict_sum_on_left => record_sum(session, :conflict, :left),
+      :conflict_sum_on_right => record_sum(session, :conflict, :left),
+      :left_on_left => record_quantity(session, :left, :left),
+      :left_on_right => record_quantity(session, :left, :left),
+      :right_on_left => 0,
+      :right_on_right => 0
+    }
+    run_sync session, expected_result
+  end
+
+  it "Direct OneWaySync should sync correctly" do
+    Initializer.configuration = deep_copy(standard_config)
+    Initializer.configuration.options = {
+      :committer => :never_commit,
+      :syncer => :one_way,
+      :direction => :right,
+      :delete => true
+    }
+
+    session = Session.new
+    expected_result = {
+      :conflict_on_left => record_quantity(session, :conflict, :left),
+      :conflict_on_right => record_quantity(session, :conflict, :right),
+      :conflict_sum_on_left => record_sum(session, :conflict, :left),
+      :conflict_sum_on_right => record_sum(session, :conflict, :left),
+      :left_on_left => record_quantity(session, :left, :left),
+      :left_on_right => record_quantity(session, :left, :left),
+      :right_on_left => 0,
+      :right_on_right => 0
+    }
+    run_sync session, expected_result
+  end
+
+  it "Proxied TwoWaySync should sync correctly" do
+    ensure_proxy
+    Initializer.configuration = deep_copy(proxied_config)
+    Initializer.configuration.options = {
+      :committer => :never_commit,
+      :syncer => :two_way,
+      :sync_conflict_handling => :right_wins,
+      :proxy_block_size => 100,
+    }
+
+    session = Session.new
+    expected_result = {
+      :conflict_on_left => record_quantity(session, :conflict, :left),
+      :conflict_on_right => record_quantity(session, :conflict, :right),
+      :conflict_sum_on_left => record_sum(session, :conflict, :right),
+      :conflict_sum_on_right => record_sum(session, :conflict, :right),
+      :left_on_left => record_quantity(session, :left, :left),
+      :left_on_right => record_quantity(session, :left, :left),
+      :right_on_left => record_quantity(session, :right, :right),
+      :right_on_right => record_quantity(session, :right, :right)
+    }
+    run_sync session, expected_result
+  end
+end

data/sims/performance/performance.rake

@@ -0,0 +1,228 @@
+require 'rake'
+require 'benchmark'
+
+require File.dirname(__FILE__) + '/../sim_helper'
+
+# Prepares the database schema for the performance tests.
+def prepare_schema
+  session = RR::Session.new
+
+  [:left, :right].each do |database|
+    [:big_scan, :big_rep, :big_rep_backup].each do |table|
+      session.send(database).drop_table table rescue nil
+      session.send(database).create_table table do |t|
+        t.column :diff_type, :string
+        t.string :text1, :text2, :text3, :text4
+        t.text :text5
+        t.binary :text6
+        t.integer :number1, :number2, :number3
+        t.float :number4
+      end rescue nil
+    end
+  end
+end  
+
+BIG_SCAN_RECORD_NUMBER = 5000 # number of records to create for simulation
+BIG_SCAN_SEED = 123 # random number seed to make simulation repeatable
+
+# Percentage values for same, modified, left_only and right_only records in simulation
+BIG_SCAN_SAME = 95
+BIG_SCAN_MODIFIED = BIG_SCAN_SAME + 3
+BIG_SCAN_LEFT_ONLY = BIG_SCAN_MODIFIED + 1 # difference to 100% will be right_only records
+
+def big_scan_columns
+  @@big_scan_columns ||= nil
+  unless @@big_scan_columns
+    session = RR::Session.new
+    @@big_scan_columns = session.left.column_names('big_scan')
+  end
+  @@big_scan_columns
+end
+
+def text_columns
+  @@text_columns ||= big_scan_columns.select {|column_name| column_name =~ /^text/}
+end
+
+def number_columns
+  @@number_columns ||= big_scan_columns.select {|column_name| column_name =~ /^number/}
+end
+
+def random_attributes
+  attributes = {}
+  text_columns.each {|column_name| attributes[column_name] = "text#{rand(1000)}"}
+  number_columns.each {|column_name| attributes[column_name] = rand(1000)}
+  attributes
+end
+
+# Populates the big_scan tables with sample data.
+def populate_scan_data()
+  session = RR::Session.new
+
+  [:left, :right].each {|database| session.send(database).execute "delete from big_scan"}
+  
+  srand BIG_SCAN_SEED
+
+  puts "\nGenerating #{BIG_SCAN_RECORD_NUMBER} records in big_scan"
+  progress_bar = ProgressBar.new BIG_SCAN_RECORD_NUMBER
+  
+  (1..BIG_SCAN_RECORD_NUMBER).each do |i|
+    
+    # Updating progress bar
+    progress_bar.step
+    
+    attributes = random_attributes
+    attributes['id'] = i
+    
+    case rand(100)
+    when 0...BIG_SCAN_SAME
+      attributes['diff_type'] = 'same'
+      [:left, :right].each {|database| session.send(database).insert_record 'big_scan', attributes}
+    when BIG_SCAN_SAME...BIG_SCAN_MODIFIED
+      attributes['diff_type'] = 'conflict'
+      session.left.insert_record 'big_scan', attributes
+      
+      attribute_name = text_columns[rand(text_columns.size)]
+      attributes[attribute_name] = attributes[attribute_name] + 'modified'
+      attribute_name = number_columns[rand(number_columns.size)]
+      attributes[attribute_name] = attributes[attribute_name] + 10000
+      session.right.insert_record 'big_scan', attributes
+    when BIG_SCAN_MODIFIED...BIG_SCAN_LEFT_ONLY
+      attributes['diff_type'] = 'left'
+      session.left.insert_record 'big_scan', attributes
+    else
+      attributes['diff_type'] = 'right'
+      session.right.insert_record 'big_scan', attributes
+    end
+  end
+end
+
+BIG_REP_CHANGE_NUMBER = 5000 # number of records to create for simulation
+BIG_REP_SEED = 456 # random number seed to make simulation repeatable
+
+# Percentage values for inserts, updates and deletes in simulation
+BIG_REP_INSERT = 45
+BIG_REP_UPDATE = BIG_REP_INSERT + 30 # different to 100% will be deletes
+
+# Populates the big_rep tables with sample data and changes.
+def populate_rep_data
+  session = RR::Session.new
+  initializer = RR::ReplicationInitializer.new session
+
+  # step 1: clear change log; ensure trigger, initialize big_rep table from data in big_scan
+  [:left, :right].each do |database|
+    initializer.drop_trigger(database, 'big_rep') rescue nil
+    session.send(database).execute "delete from big_rep"
+    session.send(database).execute "insert into big_rep select * from big_scan where diff_type = 'same'"
+    session.send(database).execute "delete from rr_pending_changes"
+    initializer.create_trigger(database, 'big_rep')
+  end
+
+  # step 2: generate changes
+
+  srand BIG_REP_SEED
+
+  # Keep tracks of the record ids in each database
+  all_ids = {}
+  all_ids[:left] = session.left.select_all("select id from big_rep").map {|row| row['id']}
+  all_ids[:right] = all_ids[:left].clone
+
+  # Next available id value
+  next_id = session.left.select_one("select max(id) + 1 as id from big_rep")['id'].to_i
+
+  puts "\nGenerating #{BIG_REP_CHANGE_NUMBER} changes in big_rep"
+  progress_bar = ProgressBar.new BIG_REP_CHANGE_NUMBER
+  (1..BIG_REP_CHANGE_NUMBER).each do
+
+    # Updating progress bar
+    progress_bar.step
+    
+    database = [:left, :right].rand
+
+    case rand(100)
+    when 0...BIG_REP_INSERT
+      attributes = random_attributes
+      attributes['diff_type'] = 'insert'
+      attributes['id'] = next_id
+      all_ids[database] << next_id
+      next_id += 1
+      session.send(database).insert_record 'big_rep', attributes
+    when BIG_REP_INSERT...BIG_REP_UPDATE
+      id = all_ids[database].rand
+      attributes = session.send(database).select_one("select * from big_rep where id = '#{id}'")
+      column = number_columns[rand(number_columns.size)]
+      attributes[column] = rand(1000)
+      session.send(database).update_record 'big_rep', attributes
+    else
+      i = rand(all_ids[database].size)
+      id = all_ids[database].delete_at(i)
+      session.send(database).delete_record 'big_rep', 'id' => id
+    end
+  end
+
+  # step 3: move data into backup tables
+  [:left, :right].each do |database|
+    session.send(database).execute "delete from big_rep_backup"
+    session.send(database).execute "insert into big_rep_backup select * from big_rep"
+    session.send(database).drop_table "big_rep_pending_changes" rescue nil
+    session.send(database).execute "create table big_rep_pending_changes as select * from rr_pending_changes"
+    initializer.drop_trigger database, 'big_rep'
+    session.send(database).execute "delete from big_rep"
+    session.send(database).execute "delete from rr_pending_changes"
+  end
+end
+
+# Generates the sample data
+def populate_data
+  populate_scan_data
+  populate_rep_data
+end
+
+# Prepares the database for the performance simulations
+def prepare
+  prepare_schema
+  puts "time required: " + Benchmark.measure {populate_data}.to_s
+end
+
+namespace :sims do
+  namespace :performance do
+    desc "Prepare database"
+    task :prepare do
+      prepare
+    end
+    
+    desc "Runs the big_scan simulation"
+    task :scan do
+      Spec::Runner::CommandLine.run(
+        Spec::Runner::OptionParser.parse(
+          ['--options', "spec/spec.opts", "./sims/performance/big_scan_spec.rb"],
+          $stdout, $stderr))
+    end
+    
+    desc "Runs the big_sync simulation"
+    task :sync do
+      Spec::Runner::CommandLine.run(
+        Spec::Runner::OptionParser.parse(
+          ['--options', "spec/spec.opts", "./sims/performance/big_sync_spec.rb"],
+          $stdout, $stderr))
+    end
+
+    desc "Runs the big_rep simulation"
+    task :rep do
+      Spec::Runner::CommandLine.run(
+        Spec::Runner::OptionParser.parse(
+          ['--options', "spec/spec.opts", "./sims/performance/big_rep_spec.rb"],
+          $stdout, $stderr))
+    end
+
+    begin
+      require 'ruby-prof/task'
+      RubyProf::ProfileTask.new do |t|
+        t.test_files = FileList["./sims/performance/*_spec.rb"]
+        t.output_dir = 'profile'
+        t.printer = :flat
+        t.min_percent = 1
+      end
+    rescue LoadError
+    end
+  end
+end

data/sims/sim_helper.rb

@@ -0,0 +1,24 @@
+# A helper class to print a text progress bar.
+class ProgressBar
+  # Creates a new progress bar for a task consisting of +max+ steps.
+  # At 100% the progress bar will be +bar_length+ characters wide.
+  def initialize(max_steps, bar_length = 40)
+    @max_steps, @bar_length, @current_progress = max_steps, bar_length, 0
+    @steps_per_progress_bar_marker = @max_steps.to_f / @bar_length
+    @marker_counter = 0
+    puts "0%>#{'-' * (@bar_length - '0%>'.length - '100%>'.length)}>100%"
+  end
+  
+  # Increases progress by +number_steps+ steps.
+  # If no argument provided, increase progress by 1 step.
+  def step(number_steps = 1)
+    @current_progress+= number_steps
+    if ((@current_progress  - number_steps) / @steps_per_progress_bar_marker).to_i \
+        < (@current_progress / @steps_per_progress_bar_marker).to_i
+      @marker_counter += 1
+      putc '.' 
+      puts if @marker_counter == @bar_length # after the last marker is printed, add a new_line
+      $stdout.flush
+    end
+  end
+end