lhm-shopify 3.5.4 → 4.0.0

data/spec/integration/chunker_spec.rb

@@ -28,7 +28,7 @@ describe Lhm::Chunker do
 
       Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
 
-      slave do
+      replica do
         value(count_all(@destination.name)).must_equal(1)
       end
 
@@ -41,7 +41,7 @@ describe Lhm::Chunker do
 
       Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
 
-      slave do
+      replica do
         value(count_all(@destination.name)).must_equal(2)
       end
     end
@@ -57,7 +57,7 @@ describe Lhm::Chunker do
 
       Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
 
-      slave do
+      replica do
         value(count_all(destination.name)).must_equal(2)
       end
     end
@@ -127,7 +127,7 @@ describe Lhm::Chunker do
 
       Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
 
-      slave do
+      replica do
         value(count_all(@destination.name)).must_equal(0)
       end
 
@@ -136,20 +136,17 @@ describe Lhm::Chunker do
     it 'should copy 23 rows from origin to destination in one shot, regardless of the value of the id' do
       23.times { |n| execute("insert into origin set id = '#{ n * n + 23 }'") }
 
-      printer = MiniTest::Mock.new
-      printer.expect(:notify, :return_value, [Integer, Integer])
-      printer.expect(:end, :return_value, [])
+      printer = mock("printer")
+      printer.expects(:notify).with(kind_of(Integer), kind_of(Integer))
+      printer.expects(:end)
 
       Lhm::Chunker.new(
         @migration, connection, { throttler: throttler, printer: printer }
       ).run
 
-      slave do
+      replica do
         value(count_all(@destination.name)).must_equal(23)
       end
-
-      printer.verify
-
     end
 
     it 'should copy all the records of a table, even if the last chunk starts with the last record of it.' do
@@ -160,42 +157,40 @@ describe Lhm::Chunker do
         @migration, connection, { throttler: Lhm::Throttler::Time.new(stride: 10), printer: printer }
       ).run
 
-      slave do
+      replica do
         value(count_all(@destination.name)).must_equal(11)
       end
 
     end
 
-    it 'should copy 23 rows from origin to destination in one shot with slave lag based throttler, regardless of the value of the id' do
+    it 'should copy 23 rows from origin to destination in one shot with replica lag based throttler, regardless of the value of the id' do
       23.times { |n| execute("insert into origin set id = '#{ 100000 + n * n + 23 }'") }
 
-      printer = MiniTest::Mock.new
-      printer.expect(:notify, :return_value, [Integer, Integer])
-      printer.expect(:end, :return_value, [])
+      printer = mock("printer")
+      printer.expects(:notify).with(kind_of(Integer), kind_of(Integer))
+      printer.expects(:end)
 
-      Lhm::Throttler::Slave.any_instance.stubs(:slave_hosts).returns(['127.0.0.1'])
-      Lhm::Throttler::SlaveLag.any_instance.stubs(:master_slave_hosts).returns(['127.0.0.1'])
+      Lhm::Throttler::Replica.any_instance.stubs(:replica_hosts).returns(['127.0.0.1'])
+      Lhm::Throttler::ReplicaLag.any_instance.stubs(:master_replica_hosts).returns(['127.0.0.1'])
 
       Lhm::Chunker.new(
-        @migration, connection, { throttler: Lhm::Throttler::SlaveLag.new(stride: 100), printer: printer }
+        @migration, connection, { throttler: Lhm::Throttler::ReplicaLag.new(stride: 100), printer: printer }
       ).run
 
-      slave do
+      replica do
         value(count_all(@destination.name)).must_equal(23)
       end
-
-      printer.verify
     end
 
-    it 'should throttle work stride based on slave lag' do
+    it 'should throttle work stride based on replica lag' do
       15.times { |n| execute("insert into origin set id = '#{ (n * n) + 1 }'") }
 
       printer = mock()
       printer.expects(:notify).with(instance_of(Integer), instance_of(Integer)).twice
       printer.expects(:end)
 
-      throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
-      def throttler.max_current_slave_lag
+      throttler = Lhm::Throttler::ReplicaLag.new(stride: 10, allowed_lag: 0)
+      def throttler.max_current_replica_lag
         1
       end
 
@@ -203,14 +198,14 @@ describe Lhm::Chunker do
         @migration, connection, { throttler: throttler, printer: printer }
       ).run
 
-      assert_equal(Lhm::Throttler::SlaveLag::INITIAL_TIMEOUT * 2 * 2, throttler.timeout_seconds)
+      assert_equal(Lhm::Throttler::ReplicaLag::INITIAL_TIMEOUT * 2 * 2, throttler.timeout_seconds)
 
-      slave do
+      replica do
         value(count_all(@destination.name)).must_equal(15)
       end
     end
 
-    it 'should detect a single slave with no lag in the default configuration' do
+    it 'should detect a single replica with no lag in the default configuration' do
       15.times { |n| execute("insert into origin set id = '#{ (n * n) + 1 }'") }
 
       printer = mock()
@@ -218,15 +213,15 @@ describe Lhm::Chunker do
       printer.expects(:verify)
       printer.expects(:end)
 
-      Lhm::Throttler::Slave.any_instance.stubs(:slave_hosts).returns(['127.0.0.1'])
-      Lhm::Throttler::SlaveLag.any_instance.stubs(:master_slave_hosts).returns(['127.0.0.1'])
+      Lhm::Throttler::Replica.any_instance.stubs(:replica_hosts).returns(['127.0.0.1'])
+      Lhm::Throttler::ReplicaLag.any_instance.stubs(:master_replica_hosts).returns(['127.0.0.1'])
 
-      throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
+      throttler = Lhm::Throttler::ReplicaLag.new(stride: 10, allowed_lag: 0)
 
-      if master_slave_mode?
-        def throttler.slave_connection(slave)
+      if master_replica_mode?
+        def throttler.replica_connection(replica)
           config = ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
-          config[:host] = slave
+          config[:host] = replica
           config[:port] = 33007
           ActiveRecord::Base.send('mysql2_connection', config)
         end
@@ -236,10 +231,10 @@ describe Lhm::Chunker do
         @migration, connection, { throttler: throttler, printer: printer }
       ).run
 
-      assert_equal(Lhm::Throttler::SlaveLag::INITIAL_TIMEOUT, throttler.timeout_seconds)
-      assert_equal(0, throttler.send(:max_current_slave_lag))
+      assert_equal(Lhm::Throttler::ReplicaLag::INITIAL_TIMEOUT, throttler.timeout_seconds)
+      assert_equal(0, throttler.send(:max_current_replica_lag))
 
-      slave do
+      replica do
         value(count_all(@destination.name)).must_equal(15)
       end
 
@@ -261,7 +256,7 @@ describe Lhm::Chunker do
 
       assert_match "Verification failed, aborting early", exception.message
 
-      slave do
+      replica do
         value(count_all(@destination.name)).must_equal(0)
       end
     end

data/spec/integration/database.yml

@@ -3,7 +3,7 @@ master:
   user: root
   password: password
   port: 33006
-slave:
+replica:
   host: mysql-2
   user: root
   password: password

data/spec/integration/entangler_spec.rb

@@ -27,7 +27,7 @@ describe Lhm::Entangler do
         execute("insert into origin (common) values ('inserted')")
       end
 
-      slave do
+      replica do
         value(count(:destination, 'common', 'inserted')).must_equal(1)
       end
     end
@@ -39,7 +39,7 @@ describe Lhm::Entangler do
         execute("delete from origin where common = 'inserted'")
       end
 
-      slave do
+      replica do
         value(count(:destination, 'common', 'inserted')).must_equal(0)
       end
     end
@@ -50,7 +50,7 @@ describe Lhm::Entangler do
         execute("update origin set common = 'updated'")
       end
 
-      slave do
+      replica do
         value(count(:destination, 'common', 'updated')).must_equal(1)
       end
     end
@@ -60,7 +60,7 @@ describe Lhm::Entangler do
 
       execute("insert into origin (common) values ('inserted')")
 
-      slave do
+      replica do
         value(count(:destination, 'common', 'inserted')).must_equal(0)
       end
     end

data/spec/integration/integration_helper.rb

@@ -53,12 +53,12 @@ module IntegrationHelper
     )
   end
 
-  def connect_slave!
+  def connect_replica!
     connect!(
       '127.0.0.1',
-      $db_config['slave']['port'],
-      $db_config['slave']['user'],
-      $db_config['slave']['password'],
+      $db_config['replica']['port'],
+      $db_config['replica']['user'],
+      $db_config['replica']['password'],
     )
   end
 
@@ -113,12 +113,12 @@ module IntegrationHelper
     end
   end
 
-  def slave(&block)
-    if master_slave_mode?
-      connect_slave!
+  def replica(&block)
+    if master_replica_mode?
+      connect_replica!
 
-      # need to wait for the slave to catch up. a better method would be to
-      # check the master binlog position and wait for the slave to catch up
+      # need to wait for the replica to catch up. a better method would be to
+      # check the master binlog position and wait for the replica to catch up
       # to that position.
       sleep 1
     else
@@ -127,7 +127,7 @@ module IntegrationHelper
 
     yield block
 
-    if master_slave_mode?
+    if master_replica_mode?
       connect_master!
     end
   end
@@ -215,8 +215,8 @@ module IntegrationHelper
   # Environment
   #
 
-  def master_slave_mode?
-    !!ENV['MASTER_SLAVE']
+  def master_replica_mode?
+    !!ENV['MASTER_REPLICA']
   end
 
   #

data/spec/integration/lhm_spec.rb

@@ -17,7 +17,7 @@ describe Lhm do
         t.add_column(:logins, "int(12) default '0'")
       end
 
-      slave do
+      replica do
         value(table_read(:users).columns['logins']).must_equal({
           :type           => 'int(12)',
           :is_nullable    => 'YES',
@@ -35,7 +35,7 @@ describe Lhm do
         t.add_column(:logins, "int(12) default '0'")
       end
 
-      slave do
+      replica do
         value(table_read(:custom_primary_key).columns['logins']).must_equal({
           :type           => 'int(12)',
           :is_nullable    => 'YES',
@@ -53,7 +53,7 @@ describe Lhm do
         t.add_column(:logins, "int(12) default '0'")
       end
 
-      slave do
+      replica do
         value(table_read(:composite_primary_key).columns['logins']).must_equal({
           :type           => 'int(12)',
           :is_nullable    => 'YES',
@@ -82,7 +82,7 @@ describe Lhm do
           t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL AUTO_INCREMENT")
         end
 
-        slave do
+        replica do
           value(connection.primary_key('users')).must_equal(['username', 'id'])
         end
       end
@@ -104,7 +104,7 @@ describe Lhm do
       describe 'when no additional data is inserted into the table' do
 
         it 'migrates the existing data' do
-          slave do
+          replica do
             value(count_all(:permissions)).must_equal(11)
           end
         end
@@ -120,7 +120,7 @@ describe Lhm do
         end
 
         it 'migrates all data' do
-          slave do
+          replica do
             value(count_all(:permissions)).must_equal(13)
           end
         end
@@ -132,7 +132,7 @@ describe Lhm do
         t.add_column(:logins, "INT(12) DEFAULT '0'")
       end
 
-      slave do
+      replica do
         value(table_read(:users).columns['logins']).must_equal({
           :type => 'int(12)',
           :is_nullable => 'YES',
@@ -150,7 +150,7 @@ describe Lhm do
         t.add_column(:logins, "INT(12) DEFAULT '0'")
       end
 
-      slave do
+      replica do
         value(count_all(:users)).must_equal(23)
       end
     end
@@ -160,7 +160,7 @@ describe Lhm do
         t.remove_column(:comment)
       end
 
-      slave do
+      replica do
         assert_nil table_read(:users).columns['comment']
       end
     end
@@ -170,7 +170,7 @@ describe Lhm do
         t.add_index([:comment, :created_at])
       end
 
-      slave do
+      replica do
         value(index_on_columns?(:users, [:comment, :created_at])).must_equal(true)
       end
     end
@@ -180,7 +180,7 @@ describe Lhm do
         t.add_index([:comment, :created_at], :my_index_name)
       end
 
-      slave do
+      replica do
         value(index?(:users, :my_index_name)).must_equal(true)
       end
     end
@@ -190,7 +190,7 @@ describe Lhm do
         t.add_index(:group)
       end
 
-      slave do
+      replica do
         value(index_on_columns?(:users, :group)).must_equal(true)
       end
     end
@@ -200,7 +200,7 @@ describe Lhm do
         t.add_unique_index(:comment)
       end
 
-      slave do
+      replica do
         value(index_on_columns?(:users, :comment, :unique)).must_equal(true)
       end
     end
@@ -210,7 +210,7 @@ describe Lhm do
         t.remove_index([:username, :created_at])
       end
 
-      slave do
+      replica do
         value(index_on_columns?(:users, [:username, :created_at])).must_equal(false)
       end
     end
@@ -220,7 +220,7 @@ describe Lhm do
         t.remove_index([:username, :group])
       end
 
-      slave do
+      replica do
         value(index?(:users, :index_with_a_custom_name)).must_equal(false)
       end
     end
@@ -230,17 +230,27 @@ describe Lhm do
         t.remove_index(:irrelevant_column_name, :index_with_a_custom_name)
       end
 
-      slave do
+      replica do
         value(index?(:users, :index_with_a_custom_name)).must_equal(false)
       end
     end
 
+    it 'should add an index with column sizes' do
+      Lhm.change_table(:users, :atomic_switch => false) do |t|
+        t.add_index(["username(6)", "group (10)", "comment     (10)"])
+      end
+
+      replica do
+        value(index_on_columns?(:users, [:username, :group, :comment])).must_equal(true)
+      end
+    end
+
     it 'should apply a ddl statement' do
       Lhm.change_table(:users, :atomic_switch => false) do |t|
         t.ddl('alter table %s add column flag tinyint(1)' % t.name)
       end
 
-      slave do
+      replica do
         value(table_read(:users).columns['flag']).must_equal({
           :type => 'tinyint(1)',
           :is_nullable => 'YES',
@@ -256,7 +266,7 @@ describe Lhm do
         t.change_column(:comment, "varchar(20) DEFAULT 'none' NOT NULL")
       end
 
-      slave do
+      replica do
         value(table_read(:users).columns['comment']).must_equal({
           :type => 'varchar(20)',
           :is_nullable => 'NO',
@@ -274,7 +284,7 @@ describe Lhm do
         t.change_column(:id, 'int(5)')
       end
 
-      slave do
+      replica do
         value(table_read(:small_table).columns['id']).must_equal({
           :type => 'int(5)',
           :is_nullable => 'NO',
@@ -293,7 +303,7 @@ describe Lhm do
         t.rename_column(:username, :login)
       end
 
-      slave do
+      replica do
         table_data = table_read(:users)
         assert_nil table_data.columns['username']
         value(table_read(:users).columns['login']).must_equal({
@@ -318,7 +328,7 @@ describe Lhm do
         t.rename_column(:group, :fnord)
       end
 
-      slave do
+      replica do
         table_data = table_read(:users)
         assert_nil table_data.columns['group']
         value(table_read(:users).columns['fnord']).must_equal({
@@ -345,7 +355,7 @@ describe Lhm do
         t.rename_column(:username, :user_name)
       end
 
-      slave do
+      replica do
         table_data = table_read(:users)
         assert_nil table_data.columns['username']
         value(table_read(:users).columns['user_name']).must_equal({
@@ -373,7 +383,7 @@ describe Lhm do
         t.rename_column(:reference, :ref)
       end
 
-      slave do
+      replica do
         table_data = table_read(:users)
         assert_nil table_data.columns['reference']
         value(table_read(:users).columns['ref']).must_equal({
@@ -400,7 +410,7 @@ describe Lhm do
         t.rename_column(:group, :fnord)
       end
 
-      slave do
+      replica do
         table_data = table_read(:users)
         assert_nil table_data.columns['group']
         value(table_read(:users).columns['fnord']).must_equal({
@@ -425,7 +435,7 @@ describe Lhm do
         t.rename_column(:username, :user_name)
       end
 
-      slave do
+      replica do
         table_data = table_read(:users)
         assert_nil table_data.columns['username']
         value(table_read(:users).columns['user_name']).must_equal({
@@ -452,7 +462,7 @@ describe Lhm do
         t.rename_column(:username, :user_name)
       end
 
-      slave do
+      replica do
         table_data = table_read(:users)
         assert_nil table_data.columns['username']
         value(table_read(:users).columns['user_name']).must_equal({
@@ -499,7 +509,7 @@ describe Lhm do
         end
       end
 
-      slave do
+      replica do
         table_data = table_read(:users)
         assert_nil table_data.columns['fnord']
         value(table_read(:users).columns['group']).must_equal({
@@ -525,7 +535,7 @@ describe Lhm do
         t.remove_index('by')
       end
 
-      slave do
+      replica do
         value(table_read(:lines).columns).must_include 'by'
         value(table_read(:lines).columns).wont_include 'lines'
         value(index_on_columns?(:lines, ['between'], :unique)).must_equal true
@@ -554,7 +564,7 @@ describe Lhm do
 
         insert.join
 
-        slave do
+        replica do
           value(count_all(:users)).must_equal(60)
         end
       end
@@ -577,7 +587,7 @@ describe Lhm do
 
         delete.join
 
-        slave do
+        replica do
           value(count_all(:users)).must_equal(40)
         end
       end
@@ -645,13 +655,12 @@ describe Lhm do
         log_lines = @logs.string.split("\n")
 
         assert log_lines.one?{ |line| line.include?("Lost connection to MySQL, will retry to connect to same host")}
-        assert log_lines.any?{ |line| line.include?("Lost connection to MySQL server at 'reading initial communication packet'")}
         assert log_lines.one?{ |line| line.include?("LHM successfully reconnected to initial host")}
         assert log_lines.one?{ |line| line.include?("100% complete")}
 
         Lhm::ChunkInsert.remove_all_callbacks
 
-        slave do
+        replica do
           value(count_all(:users)).must_equal(100)
         end
       end

data/spec/integration/locked_switcher_spec.rb

@@ -31,7 +31,7 @@ describe Lhm::LockedSwitcher do
       switcher = Lhm::LockedSwitcher.new(@migration, connection)
       switcher.run
 
-      slave do
+      replica do
         value(data_source_exists?(@origin)).must_equal true
         value(table_read(@migration.archive_name).columns.keys).must_include 'origin'
       end
@@ -41,7 +41,7 @@ describe Lhm::LockedSwitcher do
       switcher = Lhm::LockedSwitcher.new(@migration, connection)
       switcher.run
 
-      slave do
+      replica do
         value(data_source_exists?(@destination)).must_equal false
         value(table_read(@origin.name).columns.keys).must_include 'destination'
       end

data/spec/integration/sql_retry/retry_with_proxysql_spec.rb

@@ -38,7 +38,7 @@ describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
       end
     end
     assert_equal Lhm::Error, e.class
-    assert_match(/LHM tried the reconnection procedure but failed. Latest error:/, e.message)
+    assert_match(/LHM tried the reconnection procedure but failed. Aborting/, e.message)
   end
 
   it "Will retry until connection is achieved" do
@@ -61,6 +61,7 @@ describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
   it "Will abort if new writer is not same host" do
     # The hostname will be constant before the blip
     Lhm::SqlRetry.any_instance.stubs(:hostname).returns("mysql-1").then.returns("mysql-2")
+    Lhm::SqlRetry.any_instance.stubs(:server_id).returns(1).then.returns(2)
 
     # Need new instance for stub to take into effect
     lhm_retry = Lhm::SqlRetry.new(@connection, retry_options: {},
@@ -76,12 +77,12 @@ describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
     end
 
     assert_equal e.class, Lhm::Error
-    assert_match(/LHM tried the reconnection procedure but failed. Latest error: Reconnected to wrong host/, e.message)
+    assert_match(/LHM tried the reconnection procedure but failed. Aborting/, e.message)
 
     logs = @logger.string.split("\n")
 
     assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
-    assert logs.last.include?("Lost connection to MySQL server at 'reading initial communication packet")
+    assert logs.last.include?("Reconnected to wrong host. Started migration on: mysql-1 (server_id: 1), but reconnected to: mysql-2 (server_id: 2).")
   end
 
   it "Will abort if failover happens (mimicked with proxySQL)" do
@@ -98,11 +99,11 @@ describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
     end
 
     assert_equal e.class, Lhm::Error
-    assert_match(/LHM tried the reconnection procedure but failed. Latest error: Reconnected to wrong host/, e.message)
+    assert_match(/LHM tried the reconnection procedure but failed. Aborting/, e.message)
 
     logs = @logger.string.split("\n")
 
     assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
-    assert logs.last.include?("Lost connection to MySQL server at 'reading initial communication packet")
+    assert logs.last.include?("Reconnected to wrong host. Started migration on: mysql-1 (server_id: 1), but reconnected to: mysql-2 (server_id: 2).")
   end
 end

data/spec/integration/toxiproxy_helper.rb

@@ -7,7 +7,7 @@ module ToxiproxyHelper
     def included(base)
       Toxiproxy.reset
 
-      # listen on localhost, but toxiproxy is in a container itself, thus the upstream uses the Docker-Compose DNS
+      # listen on localhost, but toxiproxy is in a container itself, thus the upstream uses the Podman-Compose DNS
       Toxiproxy.populate(
         [
           {

data/spec/test_helper.rb

@@ -28,6 +28,9 @@ logger = Logger.new STDOUT
 logger.level = Logger::WARN
 Lhm.logger = logger
 
+# Want test to be efficient without having to wait the normal value of 120s
+Lhm::SqlRetry::RECONNECT_RETRY_MAX_ITERATION = 4
+
 def without_verbose(&block)
   old_verbose, $VERBOSE = $VERBOSE, nil
   yield

data/spec/unit/printer_spec.rb

@@ -70,15 +70,11 @@ describe Lhm::Printer do
     end
 
     it 'prints the dots' do
-      mock  = MiniTest::Mock.new
-      10.times do
-        mock.expect(:write, :return_value, ['.'])
-      end
+      mock  = mock("output")
+      mock.expects(:write).with('.').times(10)
 
       @printer.instance_variable_set(:@output, mock)
       10.times { @printer.notify }
-
-      mock.verify
     end
 
   end

data/spec/unit/sql_helper_spec.rb

@@ -22,7 +22,7 @@ describe Lhm::SqlHelper do
   end
 
   it 'should quote column names in index specification' do
-    value(Lhm::SqlHelper.idx_spec(['title(10)', 'album']))
-      .must_equal('`title`(10), `album`')
+    value(Lhm::SqlHelper.idx_spec(['title(10)', 'name (6)', 'album']))
+      .must_equal('`title`(10), `name`(6), `album`')
   end
 end