andyjeffries-rubyrep 1.2.1

data/spec/replication_run_spec.rb

@@ -0,0 +1,443 @@
+require File.dirname(__FILE__) + '/spec_helper.rb'
+
+include RR
+
+describe ReplicationRun do
+  before(:each) do
+    Initializer.configuration = standard_config
+  end
+
+  it "initialize should store the provided session" do
+    session = Session.new
+    sweeper = TaskSweeper.new 1
+    run = ReplicationRun.new session, sweeper
+    run.session.should == session
+  end
+
+  it "install_sweeper should install a task sweeper into the database connections" do
+    session = Session.new
+    sweeper = TaskSweeper.new 1
+    run = ReplicationRun.new session, sweeper
+    run.install_sweeper
+
+    session.left.sweeper.should == sweeper
+    session.right.sweeper.should == sweeper
+  end
+
+  it "helper should return the correctly initialized replication helper" do
+    run = ReplicationRun.new Session.new, TaskSweeper.new(1)
+    run.helper.should be_an_instance_of(ReplicationHelper)
+    run.helper.replication_run.should == run
+    run.helper.should == run.helper # ensure the helper is created only once
+  end
+
+  it "replicator should return the configured replicator" do
+    session = Session.new
+    run = ReplicationRun.new session, TaskSweeper.new(1)
+    run.replicator.
+      should be_an_instance_of(Replicators.replicators[session.configuration.options[:replicator]])
+    run.replicator.should == run.replicator # should only create the replicator once
+    run.replicator.rep_helper.should == run.helper
+  end
+
+  it "event_filtered? should behave correctly" do
+    begin
+      config = deep_copy(standard_config)
+      config.options[:committer] = :never_commit
+      session = Session.new(config)
+
+      session.left.insert_record 'extender_no_record', {
+        'id' => '1',
+        'name' => 'bla'
+      }
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'extender_no_record',
+        'change_key' => 'id|1',
+        'change_type' => 'I',
+        'change_time' => Time.now
+      }
+
+      loaders = LoggedChangeLoaders.new(session)
+      loaders.update
+      diff = ReplicationDifference.new loaders
+      diff.load
+
+      # No event filter at all
+      run = ReplicationRun.new session, TaskSweeper.new(1)
+      run.event_filtered?(diff).should be_false
+
+      # Event filter that does not handle replication events
+      session.configuration.options[:event_filter] = Object.new
+      run = ReplicationRun.new session, TaskSweeper.new(1)
+      run.event_filtered?(diff).should be_false
+
+      # event_filtered? should signal filtering (i. e. return true) if filter returns false.
+      filter = Object.new
+      def filter.before_replicate(table, key, helper, diff)
+        false
+      end
+      session.configuration.options[:event_filter] = filter
+      run = ReplicationRun.new session, TaskSweeper.new(1)
+      run.event_filtered?(diff).should be_true
+
+      # event_filtered? should return false if filter returns true.
+      filter = {}
+      def filter.before_replicate(table, key, helper, diff)
+        self[:args] = [table, key, helper, diff]
+        true
+      end
+      session.configuration.options[:event_filter] = filter
+      run = ReplicationRun.new session, TaskSweeper.new(1)
+      run.event_filtered?(diff).should be_false
+      filter[:args].should == ['extender_no_record', {'id' => 1}, run.helper, diff]
+    ensure
+      Committers::NeverCommitter.rollback_current_session
+      if session
+        session.left.execute "delete from extender_no_record"
+        session.right.execute "delete from extender_no_record"
+        session.left.execute "delete from rr_pending_changes"
+      end
+    end
+  end
+
+  it "run should replicate all logged changes" do
+    begin
+      config = deep_copy(standard_config)
+      config.options[:committer] = :never_commit
+
+      session = Session.new(config)
+
+      session.left.insert_record 'extender_no_record', {
+        'id' => '1',
+        'name' => 'bla'
+      }
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'extender_no_record',
+        'change_key' => 'id|1',
+        'change_type' => 'I',
+        'change_time' => Time.now
+      }
+
+      run = ReplicationRun.new session, TaskSweeper.new(1)
+      run.run
+
+      session.right.select_record(:table => "extender_no_record").should == {
+        'id' => 1,
+        'name' => 'bla'
+      }
+    ensure
+      Committers::NeverCommitter.rollback_current_session
+      if session
+        session.left.execute "delete from extender_no_record"
+        session.right.execute "delete from extender_no_record"
+        session.left.execute "delete from rr_pending_changes"
+      end
+    end
+  end
+
+  it "run should replication records with foreign key constraints" do
+    begin
+      config = deep_copy(standard_config)
+      config.options[:committer] = :never_commit
+
+      session = Session.new(config)
+
+      session.left.insert_record 'referencing_table', {
+        'id' => '5',
+      }
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'referencing_table',
+        'change_key' => 'id|5',
+        'change_type' => 'I',
+        'change_time' => Time.now
+      }
+
+      session.left.insert_record 'referenced_table2', {
+        'id' => '6',
+      }
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'referenced_table2',
+        'change_key' => 'id|6',
+        'change_type' => 'I',
+        'change_time' => Time.now
+      }
+
+      session.left.update_record 'referencing_table', {
+        'id' => 5,
+        'third_fk' => '6'
+      }
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'referencing_table',
+        'change_key' => 'id|5',
+        'change_new_key' => 'id|5',
+        'change_type' => 'U',
+        'change_time' => Time.now
+      }
+
+      run = ReplicationRun.new session, TaskSweeper.new(1)
+      run.run
+
+      session.right.select_record(:table => "referencing_table", :from => {'id' => 5}).should == {
+        'id' => 5,
+        'first_fk' => nil,
+        'second_fk' => nil,
+        'third_fk' => 6
+      }
+    ensure
+      Committers::NeverCommitter.rollback_current_session
+      if session
+        session.left.execute "delete from referencing_table where id = 5"
+        session.left.execute "delete from referenced_table2 where id = 6"
+
+        session.right.execute "delete from referencing_table where id = 5"
+        session.right.execute "delete from referenced_table2 where id = 6"
+        
+        session.left.execute "delete from rr_pending_changes"
+      end
+    end
+  end
+
+  it "run should not replicate filtered changes" do
+    begin
+      config = deep_copy(standard_config)
+      config.options[:committer] = :never_commit
+
+      filter = Object.new
+      def filter.before_replicate(table, key, helper, diff)
+        key['id'] != 1
+      end
+      config.options[:event_filter] = filter
+
+      session = Session.new(config)
+
+      session.left.insert_record 'extender_no_record', {
+        'id' => '1',
+        'name' => 'bla'
+      }
+      session.left.insert_record 'extender_no_record', {
+        'id' => '2',
+        'name' => 'blub'
+      }
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'extender_no_record',
+        'change_key' => 'id|1',
+        'change_type' => 'I',
+        'change_time' => Time.now
+      }
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'extender_no_record',
+        'change_key' => 'id|2',
+        'change_type' => 'I',
+        'change_time' => Time.now
+      }
+
+      run = ReplicationRun.new session, TaskSweeper.new(1)
+      run.run
+
+      session.right.select_records(:table => "extender_no_record").should == [{
+        'id' => 2,
+        'name' => 'blub'
+      }]
+    ensure
+      Committers::NeverCommitter.rollback_current_session
+      if session
+        session.left.execute "delete from extender_no_record"
+        session.right.execute "delete from extender_no_record"
+        session.left.execute "delete from rr_pending_changes"
+      end
+    end
+  end
+
+  it "run should not create the replicator if there are no pending changes" do
+    session = Session.new
+    run = ReplicationRun.new session, TaskSweeper.new(1)
+    run.should_not_receive(:replicator)
+    run.run
+  end
+
+  it "run should only replicate real differences" do
+    session = Session.new
+    session.left.begin_db_transaction
+    session.right.begin_db_transaction
+    begin
+
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'extender_no_record',
+        'change_key' => 'id|1',
+        'change_type' => 'D',
+        'change_time' => Time.now
+      }
+      session.right.insert_record 'rr_pending_changes', {
+        'change_table' => 'extender_no_record',
+        'change_key' => 'id|1',
+        'change_type' => 'D',
+        'change_time' => Time.now
+      }
+
+      run = ReplicationRun.new session, TaskSweeper.new(1)
+      run.replicator.should_not_receive(:replicate)
+      run.run
+
+    ensure
+      session.left.rollback_db_transaction
+      session.right.rollback_db_transaction
+    end
+  end
+
+  it "run should log raised exceptions" do
+    session = Session.new
+    session.left.begin_db_transaction
+    session.right.begin_db_transaction
+    begin
+      session.left.execute "delete from rr_pending_changes"
+      session.left.execute "delete from rr_logged_events"
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'extender_no_record',
+        'change_key' => 'id|1',
+        'change_type' => 'D',
+        'change_time' => Time.now
+      }
+      run = ReplicationRun.new session, TaskSweeper.new(1)
+      run.replicator.stub!(:replicate_difference).and_return {raise Exception, 'dummy message'}
+      run.run
+
+      row = session.left.select_one("select * from rr_logged_events")
+      row['description'].should == 'dummy message'
+      row['long_description'].should =~ /Exception/
+    ensure
+      session.left.rollback_db_transaction
+      session.right.rollback_db_transaction
+    end
+  end
+
+  it "run should re-raise original exception if logging to database fails" do
+    session = Session.new
+    session.left.begin_db_transaction
+    session.right.begin_db_transaction
+    begin
+      session.left.execute "delete from rr_pending_changes"
+      session.left.execute "delete from rr_logged_events"
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'extender_no_record',
+        'change_key' => 'id|1',
+        'change_type' => 'D',
+        'change_time' => Time.now
+      }
+      run = ReplicationRun.new session, TaskSweeper.new(1)
+      run.replicator.stub!(:replicate_difference).and_return {raise Exception, 'dummy message'}
+      run.helper.stub!(:log_replication_outcome).and_return {raise Exception, 'blub'}
+      lambda {run.run}.should raise_error(Exception, 'dummy message')
+    ensure
+      session.left.rollback_db_transaction
+      session.right.rollback_db_transaction
+    end
+  end
+
+  it "run should return silently if timed out before work actually started" do
+    session = Session.new
+    session.left.begin_db_transaction
+    session.right.begin_db_transaction
+    begin
+      session.left.execute "delete from rr_pending_changes"
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'extender_no_record',
+        'change_key' => 'id|1',
+        'change_type' => 'D',
+        'change_time' => Time.now
+      }
+      sweeper = TaskSweeper.new(1)
+      sweeper.stub!(:terminated?).and_return(true)
+      run = ReplicationRun.new session, sweeper
+      LoggedChangeLoaders.should_not_receive(:new)
+      run.run
+    ensure
+      session.left.rollback_db_transaction
+      session.right.rollback_db_transaction
+    end
+  end
+
+  it "run should rollback if timed out" do
+    session = Session.new
+    session.left.begin_db_transaction
+    session.right.begin_db_transaction
+    begin
+      session.left.execute "delete from rr_pending_changes"
+      session.left.execute "delete from rr_logged_events"
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'extender_no_record',
+        'change_key' => 'id|1',
+        'change_type' => 'D',
+        'change_time' => Time.now
+      }
+      sweeper = TaskSweeper.new(1)
+      sweeper.should_receive(:terminated?).any_number_of_times.and_return(false, true)
+      run = ReplicationRun.new session, sweeper
+      run.helper.should_receive(:finalize).with(false)
+      run.run
+    ensure
+      session.left.rollback_db_transaction if session.left
+      session.right.rollback_db_transaction if session.right
+    end
+  end
+
+  it "run should not catch exceptions raised during replicator initialization" do
+    config = deep_copy(standard_config)
+    config.options[:logged_replication_events] = [:invalid_option]
+    session = Session.new config
+    session.left.begin_db_transaction
+    begin
+
+      session.left.insert_record 'rr_pending_changes', {
+        'change_table' => 'extender_no_record',
+        'change_key' => 'id|1',
+        'change_type' => 'D',
+        'change_time' => Time.now
+      }
+
+      run = ReplicationRun.new session, TaskSweeper.new(1)
+      lambda {run.run}.should raise_error(ArgumentError)
+    ensure
+      session.left.rollback_db_transaction
+    end
+  end
+
+  it "run should process trigger created change log records" do
+    begin
+      config = deep_copy(standard_config)
+      config.options[:committer] = :never_commit
+      config.options[:logged_replication_events] = [:all_changes]
+
+      session = Session.new(config)
+      session.left.execute "delete from rr_logged_events"
+      initializer = ReplicationInitializer.new(session)
+      initializer.create_trigger :left, 'extender_no_record'
+
+      session.left.insert_record 'extender_no_record', {
+        'id' => '1',
+        'name' => 'bla'
+      }
+
+      run = ReplicationRun.new session, TaskSweeper.new(1)
+      run.run
+
+      session.right.select_record(:table => "extender_no_record").should == {
+        'id' => 1,
+        'name' => 'bla'
+      }
+
+      # also verify that event was logged
+      row = session.left.select_one("select * from rr_logged_events")
+      row['diff_type'].should == 'left'
+      row['change_key'].should == '1'
+      row['description'].should == 'replicated'
+    ensure
+      Committers::NeverCommitter.rollback_current_session
+      if session
+        session.left.execute "delete from extender_no_record"
+        session.right.execute "delete from extender_no_record"
+        session.left.execute "delete from rr_pending_changes"
+      end
+      initializer.drop_trigger :left, 'extender_no_record' if initializer
+    end
+  end
+end

data/spec/replication_runner_spec.rb

@@ -0,0 +1,254 @@
+require File.dirname(__FILE__) + '/spec_helper.rb'
+
+include RR
+
+describe ReplicationRunner do
+  before(:each) do
+    Initializer.configuration = standard_config
+  end
+
+  it "should register itself with CommandRunner" do
+    CommandRunner.commands['replicate'][:command].should == ReplicationRunner
+    CommandRunner.commands['replicate'][:description].should be_an_instance_of(String)
+  end
+
+  it "process_options should make options as nil and teturn status as 1 if command line parameters are unknown" do
+    # also verify that an error message is printed
+    $stderr.should_receive(:puts).any_number_of_times
+    runner = ReplicationRunner.new
+    status = runner.process_options ["--nonsense"]
+    runner.options.should == nil
+    status.should == 1
+  end
+
+  it "process_options should make options as nil and return status as 1 if config option is not given" do
+    # also verify that an error message is printed
+    $stderr.should_receive(:puts).any_number_of_times
+    runner = ReplicationRunner.new
+    status = runner.process_options []
+    runner.options.should == nil
+    status.should == 1
+  end
+
+  it "process_options should make options as nil and return status as 0 if command line includes '--help'" do
+    # also verify that the help message is printed
+    $stderr.should_receive(:puts)
+    runner = ReplicationRunner.new
+    status = runner.process_options ["--help"]
+    runner.options.should == nil
+    status.should == 0
+  end
+
+  it "process_options should set the correct options" do
+    runner = ReplicationRunner.new
+    runner.process_options ["-c", "config_path"]
+    runner.options[:config_file].should == 'config_path'
+  end
+
+  it "run should not start a replication if the command line is invalid" do
+    $stderr.should_receive(:puts).any_number_of_times
+    ReplicationRunner.any_instance_should_not_receive(:execute) {
+      ReplicationRunner.run(["--nonsense"])
+    }
+  end
+
+  it "run should start a replication if the command line is correct" do
+    ReplicationRunner.any_instance_should_receive(:execute) {
+      ReplicationRunner.run(["--config=path"])
+    }
+  end
+
+  it "session should create and return the session" do
+    runner = ReplicationRunner.new
+    runner.options = {:config_file => "config/test_config.rb"}
+    runner.session.should be_an_instance_of(Session)
+    runner.session.should == runner.session # should only be created one time
+  end
+
+  it "pause_replication should not pause if next replication is already overdue" do
+    runner = ReplicationRunner.new
+    runner.stub!(:session).and_return(Session.new(standard_config))
+    waiter_thread = mock('waiter_thread')
+    waiter_thread.should_not_receive(:join)
+    runner.instance_variable_set(:@waiter_thread, waiter_thread)
+
+    runner.pause_replication # verify no wait during first run
+    runner.instance_variable_set(:@last_run, 1.hour.ago)
+    runner.pause_replication # verify no wait if overdue
+  end
+
+  it "pause_replication should pause for correct time frame" do
+    runner = ReplicationRunner.new
+    runner.stub!(:session).and_return(Session.new(deep_copy(standard_config)))
+    runner.session.configuration.stub!(:options).and_return(:replication_interval => 2)
+    waiter_thread = mock('waiter_thread')
+    runner.instance_variable_set(:@waiter_thread, waiter_thread)
+
+    now = Time.now
+    Time.stub!(:now).and_return(now)
+    runner.instance_variable_set(:@last_run, now - 1.seconds)
+    waiter_thread.should_receive(:join).and_return {|time| time.to_f.should be_close(1.0, 0.01); 0}
+
+    runner.pause_replication
+  end
+
+  it "init_waiter should setup correct signal processing" do
+    org_stdout = $stdout
+    $stdout = StringIO.new
+    begin
+      runner = ReplicationRunner.new
+      runner.stub!(:session).and_return(Session.new(standard_config))
+    
+      # simulate sending the TERM signal
+      Signal.should_receive(:trap).with('TERM').and_yield
+
+      # also verify that the INT signal is trapped
+      Signal.should_receive(:trap).with('INT')
+
+      runner.init_waiter
+
+      # verify the that any pause would have been prematurely finished and
+      # termination signal been set
+      runner.termination_requested.should be_true
+      runner.instance_variable_get(:@waiter_thread).should_not be_alive
+      $stdout.string.should =~ /TERM.*shutdown/
+    ensure
+      $stdout = org_stdout
+    end
+  end
+
+  it "prepare_replication should call ReplicationInitializer#prepare_replication" do
+    runner = ReplicationRunner.new
+    runner.stub!(:session).and_return(:dummy_session)
+    initializer  = mock('replication_initializer')
+    initializer.should_receive(:prepare_replication)
+    ReplicationInitializer.should_receive(:new).with(:dummy_session).and_return(initializer)
+    runner.prepare_replication
+  end
+
+  # Checks a specified number of times with specified waiting period between
+  # attempts if a given SQL query returns records.
+  # Returns +true+ if a record was found
+  # * +session+: an active Session object
+  # * +database+: either :+left+ or :+right+
+  # * +query+: sql query to execute
+  # * +max_attempts+: number of attempts to find the record
+  # * +interval+: waiting time in seconds between attempts
+  def check_for_record(session, database, query, max_attempts, interval)
+    found = false
+
+    max_attempts.times do
+      found = !!session.send(database).select_one(query)
+      break if found
+      sleep interval
+    end
+    
+    found
+  end
+
+  it "execute should catch and print exceptions" do
+    org_stderr = $stderr
+    $stderr = StringIO.new
+    begin
+      session = Session.new
+      runner = ReplicationRunner.new
+      runner.stub!(:session).and_return(session)
+      runner.stub!(:init_waiter)
+      runner.stub!(:prepare_replication)
+      runner.stub!(:pause_replication)
+      runner.should_receive(:termination_requested).twice.and_return(false, true)
+
+      session.should_receive(:refresh).and_return {raise "refresh failed"}
+
+      runner.execute
+      
+      $stderr.string.should =~ /Exception caught.*refresh failed/
+      $stderr.string.should =~ /replication_runner.rb:[0-9]+:in/
+    ensure
+      $stderr = org_stderr
+    end
+  end
+
+  it "execute_once should not clean up if successful" do
+    runner = ReplicationRunner.new
+    session = Session.new
+    runner.instance_variable_set(:@session, session)
+
+    runner.execute_once
+    runner.instance_variable_get(:@session).should == session
+  end
+
+  it "execute_once should clean up after failed replication runs" do
+    runner = ReplicationRunner.new
+    session = Session.new
+    runner.instance_variable_set(:@session, session)
+
+    session.should_receive(:refresh).and_raise('bla')
+    lambda {runner.execute_once}.should raise_error('bla')
+    runner.instance_variable_get(:@session).should be_nil
+  end
+
+  it "execute_once should raise exception if replication run times out" do
+    session = Session.new
+    runner = ReplicationRunner.new
+    runner.stub!(:session).and_return(session)
+    terminated = mock("terminated")
+    terminated.stub!(:terminated?).and_return(true)
+    TaskSweeper.stub!(:timeout).and_return(terminated)
+
+    lambda {runner.execute_once}.should raise_error(/timed out/)
+  end
+
+  it "execute should start the replication" do
+    config = deep_copy(standard_config)
+    config.options[:committer] = :buffered_commit
+    config.options[:replication_interval] = 0.01
+
+    # reset table selection
+    config.included_table_specs.replace ['scanner_left_records_only']
+    config.tables_with_options.clear
+    
+    session = Session.new config
+    org_stdout = $stdout
+    begin
+      $stdout = StringIO.new
+      runner = ReplicationRunner.new
+      runner.process_options ["-c", "./config/test_config.rb"]
+      runner.stub!(:session).and_return(session)
+
+      t = Thread.new {runner.execute}
+
+      # verify that the initial sync is done
+      found = check_for_record(
+        session, :right,
+        "select * from scanner_left_records_only where id = 1",
+        100, 0.01
+      )
+      t.should be_alive # verify that the replication thread didn't die
+      found.should be_true
+
+      session.left.execute "insert into scanner_left_records_only(name) values('bla')"
+
+      # verify that the replication works
+      check_for_record(
+        session, :right,
+        "select * from scanner_left_records_only where name = 'bla'",
+        100, 0.01
+      ).should be_true
+
+      runner.instance_variable_set(:@termination_requested, true)
+      t.join
+    ensure
+      $stdout = org_stdout
+      initializer = ReplicationInitializer.new session
+      [:left, :right].each do |database|
+        initializer.clear_sequence_setup database, 'scanner_left_records_only'
+        if initializer.trigger_exists?(database, 'scanner_left_records_only')
+          initializer.drop_trigger database, 'scanner_left_records_only'
+        end
+        session.send(database).execute "delete from scanner_left_records_only where name = 'bla'"
+      end
+      session.right.execute "delete from scanner_left_records_only"
+    end
+  end
+end