lhm-shopify 3.3.5

data/spec/unit/chunk_finder_spec.rb

@@ -0,0 +1,73 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
+
+describe Lhm::ChunkFinder do
+  before(:each) do
+    @origin = Lhm::Table.new('foo')
+    @destination = Lhm::Table.new('bar')
+    @migration = Lhm::Migration.new(@origin, @destination)
+    @connection = mock()
+  end
+
+  describe '#validate' do
+    describe 'when start is greater than limit' do
+      it 'raises' do
+        assert_raises { Lhm::ChunkFinder.new(@connection, @migration, {start: 2, limit: 1}).validate }
+      end
+    end
+
+    describe 'when start is greater than limit' do
+      it 'does not raise' do
+        Lhm::ChunkFinder.new(@connection, @migration, {start: 1, limit: 2}).validate # does not raise
+      end
+    end
+  end
+
+  describe '#start' do
+    describe 'when initialized with 5' do
+      before(:each) do
+        @instance = Lhm::ChunkFinder.new(@connection, @migration, {start: 5, limit: 6})
+      end
+
+      it 'returns 5' do
+        assert_equal @instance.start, 5
+      end
+    end
+
+    describe 'when initialized with nil and the min(id) is 22' do
+      before(:each) do
+        @connection.expects(:select_value).returns(22)
+        @instance = Lhm::ChunkFinder.new(@migration, @connection, {limit: 6})
+      end
+
+      it 'returns 22' do
+        assert_equal @instance.start, 22
+      end
+    end
+  end
+
+  describe '#limit' do
+    describe 'when initialized with 6' do
+      before(:each) do
+        @instance = Lhm::ChunkFinder.new(@connection, @migration, {start: 5, limit: 6})
+      end
+
+      it 'returns 6' do
+        assert_equal @instance.limit, 6
+      end
+    end
+
+    describe 'when initialized with nil and the max(id) is 33' do
+      before(:each) do
+        @connection.expects(:select_value).returns(33)
+        @instance = Lhm::ChunkFinder.new(@migration, @connection, {start: 5})
+      end
+
+      it 'returns 33' do
+        assert_equal @instance.limit, 33
+      end
+    end
+  end
+end

data/spec/unit/chunk_insert_spec.rb

@@ -0,0 +1,44 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
+
+require 'lhm/chunk_insert'
+
+describe Lhm::ChunkInsert do
+  before(:each) do
+    @connection = stub(:connection)
+    @origin = Lhm::Table.new('foo')
+    @destination = Lhm::Table.new('bar')
+  end
+
+  describe "#sql" do
+    describe "when migration has no conditions" do
+      before { @migration = Lhm::Migration.new(@origin, @destination) }
+
+      it "uses a simple where clause" do
+        assert_equal(
+          Lhm::ChunkInsert.new(@migration, @connection, 1, 2).sql,
+          "insert ignore into `bar` () select  from `foo` where `foo`.`id` between 1 and 2"
+        )
+      end
+    end
+
+    describe "when migration has a WHERE condition" do
+      before do
+        @migration = Lhm::Migration.new(
+          @origin,
+          @destination,
+          "where foo.created_at > '2013-07-10' or foo.baz = 'quux'"
+        )
+      end
+
+      it "combines the clause with the chunking WHERE condition" do
+        assert_equal(
+          Lhm::ChunkInsert.new(@migration, @connection, 1, 2).sql,
+          "insert ignore into `bar` () select  from `foo` where (foo.created_at > '2013-07-10' or foo.baz = 'quux') and `foo`.`id` between 1 and 2"
+        )
+      end
+    end
+  end
+end

data/spec/unit/chunker_spec.rb

@@ -0,0 +1,166 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/chunker'
+require 'lhm/throttler'
+
+describe Lhm::Chunker do
+  include UnitHelper
+
+  before(:each) do
+    @origin = Lhm::Table.new('foo')
+    @destination = Lhm::Table.new('bar')
+    @migration = Lhm::Migration.new(@origin, @destination)
+    @connection = mock()
+    # This is a poor man's stub
+    @throttler = Object.new
+    def @throttler.run
+      # noop
+    end
+    def @throttler.stride
+      1
+    end
+
+    @chunker = Lhm::Chunker.new(@migration, @connection, :throttler => @throttler,
+                                                         :start     => 1,
+                                                         :limit     => 10)
+  end
+
+  describe '#run' do
+
+    it 'detects the max id to use in the chunk using the stride and use it if it is lower than the limit' do
+      def @throttler.stride
+        5
+      end
+
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/)).returns(7)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/)).returns(21)
+      @connection.expects(:update).with(regexp_matches(/between 1 and 7/)).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 8 and 10/)).returns(2)
+
+      @chunker.run
+    end
+
+
+    it 'chunks the result set according to the stride size' do
+      def @throttler.stride
+        2
+      end
+
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/)).returns(2)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/)).returns(4)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/)).returns(6)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/)).returns(8)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/)).returns(10)
+
+      @connection.expects(:update).with(regexp_matches(/between 1 and 2/)).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 3 and 4/)).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 5 and 6/)).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 7 and 8/)).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 9 and 10/)).returns(2)
+
+      @chunker.run
+    end
+
+    it 'handles stride changes during execution' do
+      # roll our own stubbing
+      def @throttler.stride
+        @run_count ||= 0
+        @run_count = @run_count + 1
+        if @run_count > 1
+          3
+        else
+          2
+        end
+      end
+
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/)).returns(2)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/)).returns(5)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/)).returns(8)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/)).returns(nil)
+
+      @connection.expects(:update).with(regexp_matches(/between 1 and 2/)).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 3 and 5/)).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 6 and 8/)).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 9 and 10/)).returns(2)
+
+      @chunker.run
+    end
+
+    it 'correctly copies single record tables' do
+      @chunker = Lhm::Chunker.new(@migration, @connection, :throttler => @throttler,
+                                                           :start     => 1,
+                                                           :limit     => 1)
+
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/)).returns(nil)
+      @connection.expects(:update).with(regexp_matches(/between 1 and 1/)).returns(1)
+
+      @chunker.run
+    end
+
+    it 'copies the last record of a table, even it is the start of the last chunk' do
+      @chunker = Lhm::Chunker.new(@migration, @connection, :throttler => @throttler,
+                                                           :start     => 2,
+                                                           :limit     => 10)
+      def @throttler.stride
+        2
+      end
+
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/)).returns(3)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/)).returns(5)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/)).returns(7)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/)).returns(9)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/)).returns(nil)
+
+      @connection.expects(:update).with(regexp_matches(/between 2 and 3/)).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 4 and 5/)).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 6 and 7/)).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 8 and 9/)).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 10 and 10/)).returns(1)
+
+      @chunker.run
+    end
+
+
+    it 'separates filter conditions from chunking conditions' do
+      @chunker = Lhm::Chunker.new(@migration, @connection, :throttler => @throttler,
+                                                           :start     => 1,
+                                                           :limit     => 2)
+      def @throttler.stride
+        2
+      end
+
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/)).returns(2)
+      @connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/)).returns(1)
+
+      def @migration.conditions
+        "where foo.created_at > '2013-07-10' or foo.baz = 'quux'"
+      end
+
+      @chunker.run
+    end
+
+    it "doesn't mess with inner join filters" do
+      @chunker = Lhm::Chunker.new(@migration, @connection, :throttler => @throttler,
+                                                           :start     => 1,
+                                                           :limit     => 2)
+
+      def @throttler.stride
+        2
+      end
+
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/)).returns(2)
+      @connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/)).returns(1)
+
+      def @migration.conditions
+        'inner join bar on foo.id = bar.foo_id'
+      end
+
+      @chunker.run
+    end
+  end
+end

data/spec/unit/entangler_spec.rb

@@ -0,0 +1,124 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
+
+require 'lhm/table'
+require 'lhm/migration'
+require 'lhm/entangler'
+
+describe Lhm::Entangler do
+  include UnitHelper
+
+  before(:each) do
+    @origin = Lhm::Table.new('origin')
+    @destination = Lhm::Table.new('destination')
+    @migration = Lhm::Migration.new(@origin, @destination)
+    @entangler = Lhm::Entangler.new(@migration)
+  end
+
+  describe 'activation' do
+    before(:each) do
+      @origin.columns['info'] = { :type => 'varchar(255)' }
+      @origin.columns['tags'] = { :type => 'varchar(255)' }
+
+      @destination.columns['info'] = { :type => 'varchar(255)' }
+      @destination.columns['tags'] = { :type => 'varchar(255)' }
+    end
+
+    it 'should create insert trigger to destination table' do
+      ddl = %Q{
+        create trigger `lhmt_ins_origin`
+        after insert on `origin` for each row
+        replace into `destination` (`info`, `tags`) /* large hadron migration */
+        values (`NEW`.`info`, `NEW`.`tags`)
+      }
+
+      @entangler.entangle.must_include strip(ddl)
+    end
+
+    it 'should create an update trigger to the destination table' do
+      ddl = %Q{
+        create trigger `lhmt_upd_origin`
+        after update on `origin` for each row
+        replace into `destination` (`info`, `tags`) /* large hadron migration */
+        values (`NEW`.`info`, `NEW`.`tags`)
+      }
+
+      @entangler.entangle.must_include strip(ddl)
+    end
+
+    it 'should create a delete trigger to the destination table' do
+      ddl = %Q{
+        create trigger `lhmt_del_origin`
+        after delete on `origin` for each row
+        delete ignore from `destination` /* large hadron migration */
+        where `destination`.`id` = OLD.`id`
+      }
+
+      @entangler.entangle.must_include strip(ddl)
+    end
+
+    it 'should retry trigger creation when it hits a lock wait timeout' do
+      connection = mock()
+      tries = 1
+      @entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0, tries: tries})
+      connection.expects(:execute).times(tries).raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
+
+      assert_raises(Mysql2::Error) { @entangler.before }
+    end
+
+    it 'should not retry trigger creation with other mysql errors' do
+      connection = mock()
+      connection.expects(:execute).once.raises(Mysql2::Error, 'The MySQL server is running with the --read-only option so it cannot execute this statement.')
+
+      @entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0})
+      assert_raises(Mysql2::Error) { @entangler.before }
+    end
+
+    it 'should succesfully finish after retrying' do
+      connection = mock()
+      connection.stubs(:execute).raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction').then.returns(true)
+      @entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0})
+
+      assert @entangler.before
+    end
+
+    it 'should retry as many times as specified by configuration' do
+      connection = mock()
+      connection.expects(:execute).times(5).raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
+      @entangler = Lhm::Entangler.new(@migration, connection, retriable: {tries: 5, base_interval: 0})
+
+      assert_raises(Mysql2::Error) { @entangler.before }
+    end
+
+    describe 'super long table names' do
+      before(:each) do
+        @origin = Lhm::Table.new('a' * 64)
+        @destination = Lhm::Table.new('b' * 64)
+        @migration = Lhm::Migration.new(@origin, @destination)
+        @entangler = Lhm::Entangler.new(@migration)
+      end
+
+      it 'should use truncated names' do
+        @entangler.trigger(:ins).length.must_be :<=, 64
+        @entangler.trigger(:upd).length.must_be :<=, 64
+        @entangler.trigger(:del).length.must_be :<=, 64
+      end
+    end
+  end
+
+  describe 'removal' do
+    it 'should remove insert trigger' do
+      @entangler.untangle.must_include('drop trigger if exists `lhmt_ins_origin`')
+    end
+
+    it 'should remove update trigger' do
+      @entangler.untangle.must_include('drop trigger if exists `lhmt_upd_origin`')
+    end
+
+    it 'should remove delete trigger' do
+      @entangler.untangle.must_include('drop trigger if exists `lhmt_del_origin`')
+    end
+  end
+end

data/spec/unit/intersection_spec.rb

@@ -0,0 +1,51 @@
+# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
+# Schmidt
+
+require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
+
+require 'lhm/table'
+require 'lhm/migrator'
+
+describe Lhm::Intersection do
+  include UnitHelper
+
+  it 'should not have dropped changes' do
+    origin = Lhm::Table.new('origin')
+    origin.columns['dropped']  = varchar
+    origin.columns['retained'] = varchar
+
+    destination = Lhm::Table.new('destination')
+    destination.columns['retained'] = varchar
+
+    intersection = Lhm::Intersection.new(origin, destination)
+    intersection.destination.include?('dropped').must_equal(false)
+  end
+
+  it 'should have unchanged columns' do
+    origin = Lhm::Table.new('origin')
+    origin.columns['dropped']  = varchar
+    origin.columns['retained'] = varchar
+
+    destination = Lhm::Table.new('destination')
+    destination.columns['retained'] = varchar
+
+    intersection = Lhm::Intersection.new(origin, destination)
+    intersection.destination.must_equal(['retained'])
+  end
+
+  it 'should have renamed columns' do
+    origin = Lhm::Table.new('origin')
+    origin.columns['old_name']  = varchar
+
+    destination = Lhm::Table.new('destination')
+    destination.columns['new_name'] = varchar
+
+    intersection = Lhm::Intersection.new(origin, destination, { 'old_name' => 'new_name' })
+    intersection.origin.must_equal(['old_name'])
+    intersection.destination.must_equal(['new_name'])
+  end
+
+  def varchar
+    { :metadata => 'VARCHAR(255)' }
+  end
+end